diff --git a/controllers/utilsController.js b/controllers/utilsController.js index 1450bba..59d987f 100644 --- a/controllers/utilsController.js +++ b/controllers/utilsController.js @@ -1,4 +1,5 @@ const { promisify } = require('util') +const fastq = require('fastq') const fetch = require('node-fetch') const ffmpeg = require('fluent-ffmpeg') const MarkdownIt = require('markdown-it') @@ -190,6 +191,73 @@ const cloudflareAuth = config.cloudflare && config.cloudflare.zoneId && (config.cloudflare.apiToken || config.cloudflare.userServiceKey || (config.cloudflare.apiKey && config.cloudflare.email)) +const cloudflarePurgeCacheQueue = cloudflareAuth && fastq.promise(async chunk => { + const MAX_TRIES = 3 + const url = `https://api.cloudflare.com/client/v4/zones/${config.cloudflare.zoneId}/purge_cache` + + const result = { + success: false, + files: chunk, + errors: [] + } + + const headers = { + 'Content-Type': 'application/json' + } + if (config.cloudflare.apiToken) { + headers.Authorization = `Bearer ${config.cloudflare.apiToken}` + } else if (config.cloudflare.userServiceKey) { + headers['X-Auth-User-Service-Key'] = config.cloudflare.userServiceKey + } else if (config.cloudflare.apiKey && config.cloudflare.email) { + headers['X-Auth-Key'] = config.cloudflare.apiKey + headers['X-Auth-Email'] = config.cloudflare.email + } + + for (let i = 0; i < MAX_TRIES; i++) { + const _log = message => { + let prefix = `[CF]: ${i + 1}/${MAX_TRIES}: ${path.basename(chunk[0])}` + if (chunk.length > 1) prefix += ',\u2026' + logger.log(`${prefix}: ${message}`) + } + + try { + const purge = await fetch(url, { + method: 'POST', + body: JSON.stringify({ files: chunk }), + headers + }) + const response = await purge.json() + + const hasErrorsArray = Array.isArray(response.errors) && response.errors.length + if (hasErrorsArray) { + const rateLimit = response.errors.find(error => /rate limit/i.test(error.message)) + if (rateLimit && i < MAX_TRIES - 1) { + _log(`${rateLimit.code}: ${rateLimit.message}. Retrying in a minute\u2026`) + await new Promise(resolve => setTimeout(resolve, 60000)) + continue + } + } + + result.success = response.success + result.errors = hasErrorsArray + ? response.errors.map(error => `${error.code}: ${error.message}`) + : [] + } catch (error) { + const errorString = error.toString() + if (i < MAX_TRIES - 1) { + _log(`${errorString}. Retrying in 5 seconds\u2026`) + await new Promise(resolve => setTimeout(resolve, 5000)) + continue + } + + result.errors = [errorString] + } + break + } + + return result +}, 1) // concurrency: 1 + self.mayGenerateThumb = extname => { extname = extname.toLowerCase() return (config.uploads.generateThumbs.image && self.imageExts.includes(extname)) || @@ -659,79 +727,15 @@ self.purgeCloudflareCache = async (names, uploads, thumbs) => { // Split array into multiple arrays with max length of 30 URLs // https://api.cloudflare.com/#zone-purge-files-by-url const MAX_LENGTH = 30 - const MAX_TRIES = 3 // only for rate limit and unexpected errors const chunks = [] while (names.length) { chunks.push(names.splice(0, MAX_LENGTH)) } - const url = `https://api.cloudflare.com/client/v4/zones/${config.cloudflare.zoneId}/purge_cache` const results = [] - - await Promise.all(chunks.map(async chunk => { - const result = { - success: false, - files: chunk, - errors: [] - } - - const headers = { - 'Content-Type': 'application/json' - } - if (config.cloudflare.apiToken) { - headers.Authorization = `Bearer ${config.cloudflare.apiToken}` - } else if (config.cloudflare.userServiceKey) { - headers['X-Auth-User-Service-Key'] = config.cloudflare.userServiceKey - } else if (config.cloudflare.apiKey && config.cloudflare.email) { - headers['X-Auth-Key'] = config.cloudflare.apiKey - headers['X-Auth-Email'] = config.cloudflare.email - } - - for (let i = 0; i < MAX_TRIES; i++) { - const _log = message => { - let prefix = `[CF]: ${i + 1}/${MAX_TRIES}: ${path.basename(chunk[0])}` - if (chunk.length > 1) prefix += ',\u2026' - logger.log(`${prefix}: ${message}`) - } - - try { - const purge = await fetch(url, { - method: 'POST', - body: JSON.stringify({ files: chunk }), - headers - }) - const response = await purge.json() - - const hasErrorsArray = Array.isArray(response.errors) && response.errors.length - if (hasErrorsArray) { - const rateLimit = response.errors.find(error => /rate limit/i.test(error.message)) - if (rateLimit && i < MAX_TRIES - 1) { - _log(`${rateLimit.code}: ${rateLimit.message}. Retrying in a minute\u2026`) - await new Promise(resolve => setTimeout(resolve, 60000)) - continue - } - } - - result.success = response.success - result.errors = hasErrorsArray - ? response.errors.map(error => `${error.code}: ${error.message}`) - : [] - } catch (error) { - const errorString = error.toString() - if (i < MAX_TRIES - 1) { - _log(`${errorString}. Retrying in 5 seconds\u2026`) - await new Promise(resolve => setTimeout(resolve, 5000)) - continue - } - - result.errors = [errorString] - } - - results.push(result) - break - } - })) - + await Promise.all(chunks.map(async chunk => + results.push(await cloudflarePurgeCacheQueue.push(chunk)) + )) return results } diff --git a/package.json b/package.json index 65ac57c..6f1a38c 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "content-disposition": "~0.5.4", "express": "~4.18.1", "express-rate-limit": "~6.4.0", + "fastq": "~1.13.0", "fluent-ffmpeg": "~2.1.2", "helmet": "~5.1.0", "jszip": "~3.10.0", diff --git a/yarn.lock b/yarn.lock index ad0ba55..e72eddd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2411,7 +2411,7 @@ fastest-levenshtein@^1.0.12: resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.12.tgz#9990f7d3a88cc5a9ffd1f1745745251700d497e2" integrity sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow== -fastq@^1.6.0: +fastq@^1.6.0, fastq@~1.13.0: version "1.13.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" integrity sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==