From 2ca2fef301b2142c4becc254568a818ef5f443f4 Mon Sep 17 00:00:00 2001 From: Bobby Date: Wed, 6 Jul 2022 17:51:34 +0700 Subject: [PATCH] feat: use SimpleDataStore for album pages cache this should have better lifecycle and use less memory over time, since we can define max items in cache at the moment hard-coded to 10 cached pages (inclusive of nojs version if ever generated) --- controllers/albumsController.js | 6 +++--- controllers/authController.js | 2 +- controllers/uploadController.js | 2 +- controllers/utilsController.js | 14 +++++++++----- routes/album.js | 31 ++++++++++++++----------------- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/controllers/albumsController.js b/controllers/albumsController.js index c2c662a..1a431bf 100644 --- a/controllers/albumsController.js +++ b/controllers/albumsController.js @@ -296,7 +296,7 @@ self.disable = async (req, res, next) => { .first() .update('enabled', 0) } - utils.invalidateAlbumsCache([id]) + utils.deleteStoredAlbumRenders([id]) utils.invalidateStatsCache('albums') try { @@ -387,7 +387,7 @@ self.edit = async (req, res, next) => { await utils.db.table('albums') .where(filter) .update(update) - utils.invalidateAlbumsCache([id]) + utils.deleteStoredAlbumRenders([id]) utils.invalidateStatsCache('albums') if (req.body.requestLink) { @@ -676,7 +676,7 @@ self.addFiles = async (req, res, next) => { await utils.db.table('albums') .whereIn('id', albumids) .update('editedAt', Math.floor(Date.now() / 1000)) - utils.invalidateAlbumsCache(albumids) + utils.deleteStoredAlbumRenders(albumids) await res.json({ success: true, failed }) } catch (error) { diff --git a/controllers/authController.js b/controllers/authController.js index 20eb6d2..0dd9a22 100644 --- a/controllers/authController.js +++ b/controllers/authController.js @@ -317,7 +317,7 @@ self.deleteUser = async (req, res, next) => { await utils.db.table('albums') .whereIn('id', albumids) .del() - utils.invalidateAlbumsCache(albumids) + utils.deleteStoredAlbumRenders(albumids) // Unlink their archives await Promise.all(albums.map(async album => { diff --git a/controllers/uploadController.js b/controllers/uploadController.js index 57965fc..a8ae802 100644 --- a/controllers/uploadController.js +++ b/controllers/uploadController.js @@ -926,7 +926,7 @@ self.storeFilesToDb = async (req, res, user, infoMap) => { await utils.db.table('albums') .whereIn('id', authorizedIds) .update('editedAt', Math.floor(Date.now() / 1000)) - utils.invalidateAlbumsCache(authorizedIds) + utils.deleteStoredAlbumRenders(authorizedIds) } } diff --git a/controllers/utilsController.js b/controllers/utilsController.js index 6276a83..cf68157 100644 --- a/controllers/utilsController.js +++ b/controllers/utilsController.js @@ -12,6 +12,7 @@ const perms = require('./permissionController') const apiErrorsHandler = require('./handlers/apiErrorsHandler') const ClientError = require('./utils/ClientError') const ServerError = require('./utils/ServerError') +const SimpleDataStore = require('./utils/SimpleDataStore') const config = require('./../config') const logger = require('./../logger') @@ -59,7 +60,6 @@ const self = { thumbsSize: config.uploads.generateThumbs.size || 200, ffprobe: promisify(ffmpeg.ffprobe), - albumsCache: {}, timezoneOffset: new Date().getTimezoneOffset(), retentions: { @@ -68,6 +68,10 @@ const self = { default: {} }, + albumRenderStore: new SimpleDataStore({ + limit: 10, + strategy: SimpleDataStore.STRATEGIES[0] + }), contentDispositionStore: null } @@ -678,7 +682,7 @@ self.bulkDeleteFromDb = async (field, values, user) => { .whereIn('id', albumids) .update('editedAt', Math.floor(Date.now() / 1000)) .catch(logger.error) - self.invalidateAlbumsCache(albumids) + self.deleteStoredAlbumRenders(albumids) } // Purge Cloudflare's cache if necessary, but do not wait @@ -769,10 +773,10 @@ self.bulkDeleteExpired = async (dryrun, verbose) => { return result } -self.invalidateAlbumsCache = albumids => { +self.deleteStoredAlbumRenders = albumids => { for (const albumid of albumids) { - delete self.albumsCache[albumid] - delete self.albumsCache[`${albumid}-nojs`] + self.albumRenderStore.delete(`${albumid}`) + self.albumRenderStore.delete(`${albumid}-nojs`) } } diff --git a/routes/album.js b/routes/album.js index 104e4b8..ada6407 100644 --- a/routes/album.js +++ b/routes/album.js @@ -26,17 +26,13 @@ routes.get('/a/:identifier', async (req, res, next) => { let cacheid if (process.env.NODE_ENV !== 'development') { - // Cache ID - we initialize a separate cache for No-JS version - cacheid = nojs ? `${album.id}-nojs` : album.id + // Cache ID - we use a separate cache key for No-JS version + cacheid = `${album.id}${nojs ? '-nojs' : ''}` - if (!utils.albumsCache[cacheid]) { - utils.albumsCache[cacheid] = { - cache: null, - generating: false - } - } - - if (!utils.albumsCache[cacheid].cache && utils.albumsCache[cacheid].generating) { + const cache = utils.albumRenderStore.get(cacheid) + if (cache) { + return res.send(cache) + } else if (cache === null) { return res.render('album-notice', { config, utils, @@ -44,11 +40,9 @@ routes.get('/a/:identifier', async (req, res, next) => { album, notice: 'This album\'s public page is still being generated. Please try again later.' }) - } else if (utils.albumsCache[cacheid].cache) { - return res.send(utils.albumsCache[cacheid].cache) } - utils.albumsCache[cacheid].generating = true + utils.albumRenderStore.hold(cacheid) } const files = await utils.db.table('files') @@ -90,13 +84,16 @@ routes.get('/a/:identifier', async (req, res, next) => { }, (error, html) => { const data = error ? null : html if (cacheid) { - utils.albumsCache[cacheid].cache = data - utils.albumsCache[cacheid].generating = false + // Only store rendered page if it did not error out and album actually have files + if (data && files.length) { + utils.albumRenderStore.set(cacheid, data) + } else { + utils.albumRenderStore.delete(cacheid) + } } // Express should already send error to the next handler - if (error) return - return res.send(data) + if (!error) return res.send(data) }) })