Mirror/filesafe
1
0
Fork 0
mirror of https://github.com/BobbyWibowo/lolisafe.git synced 2024-12-14 00:16:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

try cf purge up to 3 times

Browse Source 
with preset delay in between (60 secs if rate limited, 5 secs for
unexpected errors)

an alternative to global queue mechanism in #342
not perfect, but easier to implement

closes #342
This commit is contained in:
Bobby Wibowo 2020-12-26 16:52:07 +07:00
parent 3a0810b0be
commit d7b11e7e56
No known key found for this signature in database
GPG Key ID: 51C3A1E1E22D26CF
1 changed files with 54 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
controllers/utilsController.js
View File

@ -525,8 +525,8 @@ self.purgeCloudflareCache = async (names, uploads, thumbs) => {
// Split array into multiple arrays with max length of 30 URLs
// https://api.cloudflare.com/#zone-purge-files-by-url
// TODO: Handle API rate limits
const MAX_LENGTH = 30
const MAX_TRIES = 3 // only for rate limit and unexpected errors
const chunks = []
while (names.length) {
chunks.push(names.splice(0, MAX_LENGTH))
@ -542,35 +542,61 @@ self.purgeCloudflareCache = async (names, uploads, thumbs) => {
errors: []
}
try {
const headers = {
'Content-Type': 'application/json'
}
if (config.cloudflare.apiToken) {
headers.Authorization = `Bearer ${config.cloudflare.apiToken}`
} else if (config.cloudflare.userServiceKey) {
headers['X-Auth-User-Service-Key'] = config.cloudflare.userServiceKey
} else if (config.cloudflare.apiKey && config.cloudflare.email) {
headers['X-Auth-Key'] = config.cloudflare.apiKey
headers['X-Auth-Email'] = config.cloudflare.email
}
const purge = await fetch(url, {
method: 'POST',
body: JSON.stringify({ files: chunk }),
headers
})
const response = await purge.json()
result.success = response.success
if (Array.isArray(response.errors) && response.errors.length) {
result.errors = response.errors.map(error => `${error.code}: ${error.message}`)
}
} catch (error) {
result.errors = [error.toString()]
const headers = {
'Content-Type': 'application/json'
}
if (config.cloudflare.apiToken) {
headers.Authorization = `Bearer ${config.cloudflare.apiToken}`
} else if (config.cloudflare.userServiceKey) {
headers['X-Auth-User-Service-Key'] = config.cloudflare.userServiceKey
} else if (config.cloudflare.apiKey && config.cloudflare.email) {
headers['X-Auth-Key'] = config.cloudflare.apiKey
headers['X-Auth-Email'] = config.cloudflare.email
}
results.push(result)
for (let i = 0; i < MAX_TRIES; i++) {
const _log = message => {
let prefix = `[CF]: ${i + 1}/${MAX_TRIES}: ${path.basename(chunk[0])}`
if (chunk.length > 1) prefix += ',\u2026'
logger.log(`${prefix}: ${message}`)
}
try {
const purge = await fetch(url, {
method: 'POST',
body: JSON.stringify({ files: chunk }),
headers
})
const response = await purge.json()
const hasErrorsArray = Array.isArray(response.errors) && response.errors.length
if (hasErrorsArray) {
const rateLimit = response.errors.find(error => /rate limit/i.test(error.message))
if (rateLimit && i < MAX_TRIES - 1) {
_log(`${rateLimit.code}: ${rateLimit.message}. Retrying in a minute\u2026`)
await new Promise(resolve => setTimeout(resolve, 60000))
continue
}
}
result.success = response.success
result.errors = hasErrorsArray
? response.errors.map(error => `${error.code}: ${error.message}`)
: []
} catch (error) {
const errorString = error.toString()
if (i < MAX_TRIES - 1) {
_log(`${errorString}. Retrying in 5 seconds\u2026`)
await new Promise(resolve => setTimeout(resolve, 5000))
continue
}
result.errors = [errorString]
}
results.push(result)
break
}
}))
return results