From 8a1ff434d909c04a4d5fce0dfc921563134137cd Mon Sep 17 00:00:00 2001 From: Bobby Wibowo Date: Sun, 3 Jul 2022 10:35:36 +0700 Subject: [PATCH] feat: in-memory caching of content-disposition please read config.sample.js ignore if not serving files with node or not having the option turned on --- config.sample.js | 18 +++- controllers/utils/SimpleDataStore.js | 124 +++++++++++++++++++++++++++ controllers/utilsController.js | 10 ++- lolisafe.js | 34 ++++++-- 4 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 controllers/utils/SimpleDataStore.js diff --git a/config.sample.js b/config.sample.js index 032719f..a978af2 100644 --- a/config.sample.js +++ b/config.sample.js @@ -38,11 +38,25 @@ module.exports = { /* If you serve files with node, you can optionally choose to set Content-Disposition header - into their original file names. This allows users to save files into their original file names. + with their original file names. This allows users to download files into their original file names. - This will query the DB every time users access uploaded files as there's no caching mechanism. + "contentDispositionOptions" configures in-memory caching options, + as it would otherwise have to query database every single time. + + If enabled, but "contentDispositionOptions" is missing, it will use these defaults: + { limit: 50, strategy: 'lastGetTime' } */ setContentDisposition: false, + contentDispositionOptions: { + limit: 50, + /* + Available strategies: lastGetTime, getsCount + + lastGetTime: when cache store exceeds limit, remove cache with oldest access time + getsCount: when cache store exceeds limit, remove cache with fewest access count + */ + strategy: 'lastGetTime' + }, /* If you serve files with node, you can optionally choose to diff --git a/controllers/utils/SimpleDataStore.js b/controllers/utils/SimpleDataStore.js new file mode 100644 index 0000000..e48357c --- /dev/null +++ b/controllers/utils/SimpleDataStore.js @@ -0,0 +1,124 @@ +const STRATEGIES = [ + 'lastGetTime', + 'getsCount' +] + +class SimpleDataStore { + #store + #limit + #strategy + + constructor (options = {}) { + if (typeof options !== 'object') { + throw new TypeError('Missing options object.') + } + + if (!Number.isFinite(options.limit) || options.limit <= 1) { + throw new TypeError('Limit must be a finite number that is at least 2.') + } + + if (!STRATEGIES.includes(options.strategy)) { + throw new TypeError(`Strategy must be one of these: ${STRATEGIES.map(s => `"${s}"`).join(', ')}.`) + } + + this.#store = new Map() + this.#limit = options.limit + this.#strategy = options.strategy + } + + clear () { + return this.#store.clear() + } + + delete (key) { + return this.#store.delete(key) + } + + get (key) { + const entry = this.#store.get(key) + if (typeof entry === 'undefined') return entry + + switch (this.#strategy) { + case STRATEGIES[0]: + entry.stratval = Date.now() + break + case STRATEGIES[1]: + entry.stratval++ + break + } + + this.#store.set(key, entry) + return entry.value + } + + getStalest () { + let stalest = null + switch (this.#strategy) { + // both "lastGetTime" and "getsCount" simply must find lowest value + // to determine the stalest entry + case STRATEGIES[0]: + case STRATEGIES[1]: + for (const entry of this.#store) { + if (!stalest || entry[1].stratval < stalest[1].stratval) { + stalest = entry + } + } + break + } + + // return its key only + return stalest[0] + } + + set (key, value) { + if (this.#store.size >= this.#limit) { + const stalest = this.getStalest() + if (stalest) { + this.#store.delete(stalest) + } + } + + let stratval + switch (this.#strategy) { + case STRATEGIES[0]: + stratval = Date.now() + break + case STRATEGIES[1]: + stratval = 0 + break + } + + return this.#store.set(key, { value, stratval }) && true + } + + get limit () { + return this.#limit + } + + set limit (_) { + throw Error('This property is read-only.') + } + + get size () { + return this.#store.size + } + + set size (_) { + throw Error('This property is read-only.') + } + + get strategy () { + return this.#strategy + } + + set strategy (_) { + throw Error('This property is read-only.') + } + + get store () { + // return shallow copy + return new Map(this.#store) + } +} + +module.exports = SimpleDataStore diff --git a/controllers/utilsController.js b/controllers/utilsController.js index 59d987f..6276a83 100644 --- a/controllers/utilsController.js +++ b/controllers/utilsController.js @@ -66,7 +66,9 @@ const self = { enabled: false, periods: {}, default: {} - } + }, + + contentDispositionStore: null } // Remember old renderer, if overridden, or proxy to default renderer @@ -654,11 +656,15 @@ self.bulkDeleteFromDb = async (field, values, user) => { }) } - // Push album ids unlinked.forEach(file => { + // Push album ids if (file.albumid && !albumids.includes(file.albumid)) { albumids.push(file.albumid) } + // Delete form Content-Disposition store if used + if (self.contentDispositionStore) { + self.contentDispositionStore.delete(file.name) + } }) // Push unlinked files diff --git a/lolisafe.js b/lolisafe.js index c708cf1..488e2e3 100644 --- a/lolisafe.js +++ b/lolisafe.js @@ -139,6 +139,13 @@ const overrideContentTypes = contentTypes && contentTypes.length && function (re const initServeStaticUploads = (opts = {}) => { if (config.setContentDisposition) { + const SimpleDataStore = require('./controllers/utils/SimpleDataStore') + utils.contentDispositionStore = new SimpleDataStore( + config.contentDispositionOptions || { + limit: 50, + strategy: 'lastGetTime' + } + ) opts.preSetHeaders = async (res, req, path, stat) => { try { // Do only if accessing files from uploads' root directory (i.e. not thumbs, etc.) @@ -146,20 +153,33 @@ const initServeStaticUploads = (opts = {}) => { const relpath = path.replace(paths.uploads, '') if (relpath.indexOf('/', 1) === -1 && req.method === 'GET') { const name = relpath.substring(1) - const _file = await utils.db.table('files') - .where('name', name) - .select('original') - .first() - res.set('Content-Disposition', contentDisposition(_file.original, { type: 'inline' })) + let original = utils.contentDispositionStore.get(name) + if (!original) { + original = await utils.db.table('files') + .where('name', name) + .select('original') + .first() + .then(_file => { + utils.contentDispositionStore.set(name, _file.original) + return _file.original + }) + } + if (original) { + res.set('Content-Disposition', contentDisposition(original, { type: 'inline' })) + } } } catch (error) { logger.error(error) } } // serveStatic is provided with @bobbywibowo/serve-static, a fork of express/serve-static. - // The fork allows specifying an async setHeaders function by the name preSetHeaders. - // It will await the said function before creating 'send' stream to client. + // The fork allows specifying an async function by the name preSetHeaders, + // which it will await before creating 'send' stream to client. + // This is necessary due to database queries being async tasks, + // and express/serve-static not having the functionality by default. safe.use('/', require('@bobbywibowo/serve-static')(paths.uploads, opts)) + logger.debug('Inititated SimpleDataStore for Content-Disposition: ' + + `{ limit: ${utils.contentDispositionStore.limit}, strategy: "${utils.contentDispositionStore.strategy}" }`) } else { safe.use('/', express.static(paths.uploads, opts)) }