From 8ab77a6464c9f4ca0ff1d621dd5d5a903c0ea7da Mon Sep 17 00:00:00 2001 From: Bobby Wibowo Date: Thu, 19 Sep 2019 19:10:37 +0700 Subject: [PATCH] Updated Removed version strings from _globals.njk, in favor of src/versions.json. That versions in that file can be bumped with "yarn bump-versions". v1 is automatically bumped when doing "yarn build" as well. Added README file in src directory, explaining versions.json file. Added README file in scripts directory, detailing usage of each scripts. Version strings will no longer be appended when cacheControl is disabled in config file. After all, version strings are only needed when the static assets are cached indefinitely in users' browsers. Initial Cloudflare's cache purging will no longer be executed when cloudflare -> purgeCache is disabled, even if cacheControl is enabled. Just in case someone wants to use version strings for other use cases. Actually use custom metaDesc variable on meta description tag. --- gulpfile.js | 15 +++++-- lolisafe.js | 53 +++++++++++++++++-------- package.json | 1 + routes/album.js | 8 +++- routes/nojs.js | 1 + scripts/README.md | 57 +++++++++++++++++++++++++++ scripts/bump-versions.js | 83 +++++++++++++++++++++++++++++++++++++++ scripts/clean-up.js | 3 +- scripts/delete-expired.js | 3 +- src/README.md | 31 +++++++++++++++ src/versions.json | 6 +++ todo.md | 2 +- views/_globals.njk | 13 ------ views/_layout.njk | 28 ++++++------- views/album.njk | 14 +++---- views/auth.njk | 10 ++--- views/dashboard.njk | 24 +++++------ views/home.njk | 24 +++++------ views/nojs.njk | 4 +- 19 files changed, 289 insertions(+), 91 deletions(-) create mode 100644 scripts/README.md create mode 100644 scripts/bump-versions.js create mode 100644 src/README.md create mode 100644 src/versions.json diff --git a/gulpfile.js b/gulpfile.js index 38360e6..5ff675c 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -1,3 +1,4 @@ +const { exec } = require('child_process') const gulp = require('gulp') const cssnano = require('cssnano') const del = require('del') @@ -91,7 +92,15 @@ gulp.task('build:js', () => { gulp.task('build', gulp.parallel('build:css', 'build:js')) -gulp.task('default', gulp.series('lint', 'clean', 'build')) +gulp.task('exec:bump-versions', cb => { + exec('node ./scripts/bump-versions.js 1', (error, stdout, stderr) => { + if (stdout) process.stdout.write(stdout) + if (stderr) process.stderr.write(stderr) + cb(error) + }) +}) + +gulp.task('default', gulp.series('lint', 'clean', 'build', 'exec:bump-versions')) /** TASKS: WATCH (SKIP LINTER) */ @@ -109,7 +118,7 @@ gulp.task('watch:js', () => { gulp.task('watch:src', gulp.parallel('watch:css', 'watch:js')) -gulp.task('nodemon', done => { +gulp.task('nodemon', cb => { return nodemon({ script: './lolisafe.js', env: process.env, @@ -125,7 +134,7 @@ gulp.task('nodemon', done => { 'views/album.njk' ], ext: 'js', - done + done: cb }) }) diff --git a/lolisafe.js b/lolisafe.js index 036efdf..6fef638 100644 --- a/lolisafe.js +++ b/lolisafe.js @@ -8,6 +8,7 @@ const RateLimit = require('express-rate-limit') const readline = require('readline') const config = require('./config') const logger = require('./logger') +const versions = require('./src/versions') const safe = express() process.on('uncaughtException', error => { @@ -112,14 +113,28 @@ safe.use('/api', api) process.exit(1) } + // Re-map version strings if cache control is enabled (safe.fiery.me) + utils.versionStrings = {} + if (config.cacheControl) + for (const type in versions) + utils.versionStrings[type] = `?_=${versions[type]}` + + // Check for custom pages, otherwise fallback to Nunjucks templates for (const page of config.pages) { const customPage = path.join(paths.customPages, `${page}.html`) if (!await paths.access(customPage).catch(() => true)) safe.get(`/${page === 'home' ? '' : page}`, (req, res, next) => res.sendFile(customPage)) else if (page === 'home') - safe.get('/', (req, res, next) => res.render(page, { config, gitHash: utils.gitHash })) + safe.get('/', (req, res, next) => res.render(page, { + config, + versions: utils.versionStrings, + gitHash: utils.gitHash + })) else - safe.get(`/${page}`, (req, res, next) => res.render(page, { config })) + safe.get(`/${page}`, (req, res, next) => res.render(page, { + config, + versions: utils.versionStrings + })) } // Error pages @@ -176,23 +191,27 @@ safe.use('/api', api) logger.log(`lolisafe started on port ${config.port}`) // Cache control (safe.fiery.me) - if (config.cacheControl) { - logger.log('Cache control enabled, purging...') - const routes = config.pages.concat(['api/check']) - const results = await utils.purgeCloudflareCache(routes) - let errored = false - let succeeded = 0 - for (const result of results) { - if (result.errors.length) { - if (!errored) errored = true - result.errors.forEach(error => logger.log(`[CF]: ${error}`)) - continue + // Also only if explicitly using Cloudflare + if (config.cacheControl) + if (config.cloudflare.purgeCache) { + logger.log('Cache control enabled, purging Cloudflare\'s cache...') + const routes = config.pages.concat(['api/check']) + const results = await utils.purgeCloudflareCache(routes) + let errored = false + let succeeded = 0 + for (const result of results) { + if (result.errors.length) { + if (!errored) errored = true + result.errors.forEach(error => logger.log(`[CF]: ${error}`)) + continue + } + succeeded += result.files.length } - succeeded += result.files.length + if (!errored) + logger.log(`Successfully purged ${succeeded} cache`) + } else { + logger.log('Cache control enabled without Cloudflare\'s cache purging') } - if (!errored) - logger.log(`Purged ${succeeded} Cloudflare's cache`) - } // Temporary uploads if (Array.isArray(config.uploads.temporaryUploadAges) && config.uploads.temporaryUploadAges.length) { diff --git a/package.json b/package.json index 386f58d..fcbb3cd 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "build": "gulp default", "watch": "gulp watch", "develop": "env NODE_ENV=development yarn watch", + "bump-versions": "node ./scripts/bump-versions.js", "cf-purge": "node ./scripts/cf-purge.js", "clean-up": "node ./scripts/clean-up.js", "delete-expired": "node ./scripts/delete-expired.js", diff --git a/routes/album.js b/routes/album.js index a1780e5..dc73da5 100644 --- a/routes/album.js +++ b/routes/album.js @@ -82,7 +82,13 @@ routes.get('/a/:identifier', async (req, res, next) => { album.url = `a/${album.identifier}` - return res.render('album', { config, album, files, nojs }, (error, html) => { + return res.render('album', { + config, + versions: utils.versionStrings, + album, + files, + nojs + }, (error, html) => { utils.albumsCache[cacheid].cache = error ? null : html utils.albumsCache[cacheid].generating = false diff --git a/routes/nojs.js b/routes/nojs.js index 1b18285..44200c7 100644 --- a/routes/nojs.js +++ b/routes/nojs.js @@ -13,6 +13,7 @@ routes.post('/nojs', (req, res, next) => { const result = args[0] return res.render('nojs', { config, + versions: utils.versionStrings, gitHash: utils.gitHash, errorMessage: result.success ? '' : (result.description || 'An unexpected error occurred.'), files: result.files || [{}] diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..e0f92f5 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,57 @@ +# README + +## cf-purge.js + +```none +$ yarn cf-purge +$ node ./scripts/cf-purge.js + +Purge Cloudflare's cache. + +Usage: +node scripts/cf-purge.js ...filename + +filename: +Upload names separated by space (will automatically include their thumbs if available). +``` + +## clean-up.js + +```none +$ yarn clean-up -h +$ node ./scripts/clean-up.js -h + +Clean up files that are not in the database. + +Usage: +node scripts/clean-up.js [mode=0|1|2] + +mode: +0 = Only list names of files that are not in the database. +1 = Clean up the files. +``` + +## delete-expired.js + +```none +$ yarn delete-expired -h +$ node ./scripts/delete-expired.js -h + +Bulk delete expired files. + +Usage: +node scripts/delete-expired.js [mode=0|1|2] + +mode: +0 = Only list names of the expired files. +1 = Delete expired files (output file names). +2 = Delete expired files (no output). +``` + +## thumbs.js + +[\[...\]](..#script-for-missing-thumbnails) + +## bump-versions.js + +[\[...\]](../src#readme) diff --git a/scripts/bump-versions.js b/scripts/bump-versions.js new file mode 100644 index 0000000..001616b --- /dev/null +++ b/scripts/bump-versions.js @@ -0,0 +1,83 @@ +const { promisify } = require('util') +const fs = require('fs') +const path = require('path') +const utils = require('../controllers/utilsController') + +const self = { + access: promisify(fs.access), + readFile: promisify(fs.readFile), + writeFile: promisify(fs.writeFile), + types: null +} + +;(async () => { + const location = process.argv[1].replace(process.cwd() + '/', '') + const args = process.argv.slice(2) + + self.types = {} + for (const arg of args) { + const lower = arg.toLowerCase() + if (lower === 'a') { + self.types = { 1: '', 2: '', 3: '', 4: '' } + break + } + const parsed = parseInt(lower) + // Only accept 1 to 4 + if (!isNaN(parsed) && parsed >= 1 && parsed <= 4) + self.types[parsed] = '' + } + + if (args.includes('--help') || args.includes('-h') || !Object.keys(self.types).length) + return console.log(utils.stripIndents(` + Bump version strings for client-side assets. + + Usage: + node ${location} + + types: + Space separated list of types (accepts 1 to 4). + 1: CSS and JS files (lolisafe core assets + fontello.css). + 2: Icons, images and config files (manifest.json, browserconfig.xml, etc). + 3: CSS and JS files (libs from /public/libs, such as bulma, lazyload, etc). + 4: Renders from /public/render/* directories (to be used with /src/js/misc/render.js). + a: Shortcut to update all types. + `)) + + const file = path.resolve('./src/versions.json') + + // Create an empty file if it does not exist + try { + await self.access(file) + } catch (error) { + if (error.code === 'ENOENT') + await self.writeFile(file, '{}') + else + // Re-throw error + throw error + } + + // Read & parse existing versions + const old = JSON.parse(await self.readFile(file)) + + // Bump version of selected types + // We use current timestamp cause it will always increase + const types = Object.keys(self.types) + const bumped = String(Math.floor(Date.now() / 1000)) // 1s precision + for (const type of types) + self.types[type] = bumped + + // Overwrite existing versions with new versions + const data = Object.assign(old, self.types) + + // Stringify new versions + const stringified = JSON.stringify(data, null, 2) + + // Write to file + await self.writeFile(file, stringified) + console.log(`Successfully bumped version string of type ${types.join(', ')} to "${bumped}".`) +})() + .then(() => process.exit(0)) + .catch(error => { + console.error(error) + process.exit(1) + }) diff --git a/scripts/clean-up.js b/scripts/clean-up.js index 945e7fe..e55b62a 100644 --- a/scripts/clean-up.js +++ b/scripts/clean-up.js @@ -23,8 +23,6 @@ self.getFiles = async directory => { const location = process.argv[1].replace(process.cwd() + '/', '') const args = process.argv.slice(2) - self.mode = parseInt(args[0]) || 0 - if (args.includes('--help') || args.includes('-h')) return console.log(utils.stripIndents(` Clean up files that are not in the database. @@ -37,6 +35,7 @@ self.getFiles = async directory => { 1 = Clean up the files. `)) + self.mode = parseInt(args[0]) || 0 const dryrun = self.mode === 0 const uploads = await self.getFiles(paths.uploads) diff --git a/scripts/delete-expired.js b/scripts/delete-expired.js index 3039634..4ade8ad 100644 --- a/scripts/delete-expired.js +++ b/scripts/delete-expired.js @@ -8,8 +8,6 @@ const self = { const location = process.argv[1].replace(process.cwd() + '/', '') const args = process.argv.slice(2) - self.mode = parseInt(args[0]) || 0 - if (args.includes('--help') || args.includes('-h')) return console.log(utils.stripIndents(` Bulk delete expired files. @@ -23,6 +21,7 @@ const self = { 2 = Delete expired files (no output). `)) + self.mode = parseInt(args[0]) || 0 const dryrun = self.mode === 0 const quiet = self.mode === 2 diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..d8ee5b1 --- /dev/null +++ b/src/README.md @@ -0,0 +1,31 @@ +# README + +`versions.json` is the file that tells Nunjucks what version strings to append to client-side lolisafe assets. + +To bump the version, it's recommended to use use `yarn bump-versions`. + +```none +$ yarn bump-versions +$ node ./scripts/bump-versions.js + +Bump version strings for client-side assets. + +Usage: +node scripts/bump-versions.js + +types: +Space separated list of types (accepts 1 to 4). +1: CSS and JS files (lolisafe core assets + fontello.css). +2: Icons, images and config files (manifest.json, browserconfig.xml, etc). +3: CSS and JS files (libs from /public/libs, such as bulma, lazyload, etc). +4: Renders from /public/render/* directories (to be used with /src/js/misc/render.js). +a: Shortcut to update all types. +``` + +By default, running `yarn build` will also run `node ./scripts/bump-versions.js 1`. + +## Cache-Control + +Version strings will NOT be used when `cacheControl` in `config.js` is not enabled. + +To begin with, version strings are only necessary when the assets are being cached indefinitely in browsers. diff --git a/src/versions.json b/src/versions.json new file mode 100644 index 0000000..813267a --- /dev/null +++ b/src/versions.json @@ -0,0 +1,6 @@ +{ + "1": "1568894228", + "2": "1568894058", + "3": "1568894058", + "4": "1568894058" +} \ No newline at end of file diff --git a/todo.md b/todo.md index 3fbdfb1..c17e833 100644 --- a/todo.md +++ b/todo.md @@ -4,7 +4,7 @@ Normal priority: * [x] Improve performance of album public pages, ~~and maybe paginate them~~. * [x] Use [native lazy-load tag](https://web.dev/native-lazy-loading) on nojs album pages. -* [ ] Use incremental version numbering instead of randomized strings. +* [x] Use incremental version numbering instead of randomized strings. * [ ] Use versioning in APIs, somehow. * [ ] Better `df` handling (system disk stats). * [x] Use loading spinners on dashboard's sidebar menus. diff --git a/views/_globals.njk b/views/_globals.njk index d3d4097..87c2b0b 100644 --- a/views/_globals.njk +++ b/views/_globals.njk @@ -7,19 +7,6 @@ {% set google_site_verification = null %} -{# - This will be appended to the URLs of static files (CSS, JS, images, etc), - and should be changed on every updates to make sure clients load the very latest version of them. - v1: CSS and JS files (lolisafe). - v2: Images and config files (manifest.json, browserconfig.xml, etc). - v3: CSS and JS files (libs such as bulma, lazyload, etc). - v4: Renders in /public/render/* directories (to be used by render.js). -#} -{% set v1 = "sWcNtiNy3c" %} -{% set v2 = "sWcNtiNy3c" %} -{% set v3 = "YFsTqC68Bc" %} -{% set v4 = "S3TAWpPeFS" %} - {# These will be the links in the homepage and the No-JS uploader. You may remove icons by changing "home_icons" to false. diff --git a/views/_layout.njk b/views/_layout.njk index 0ff04e4..9205af0 100644 --- a/views/_layout.njk +++ b/views/_layout.njk @@ -17,7 +17,7 @@ - + @@ -26,8 +26,8 @@ {% block stylesheets %} - - + + {% endblock %} {% block opengraph %} @@ -39,10 +39,10 @@ {%- if metaImage %} {%- endif %} - + - + @@ -54,21 +54,21 @@ {%- if metaImage %} {% else %} - + {%- endif %} {% endblock %} - - - - - - - + + + + + + + - + {% block endmeta %}{% endblock %} diff --git a/views/album.njk b/views/album.njk index 915bae0..62cffa5 100644 --- a/views/album.njk +++ b/views/album.njk @@ -14,18 +14,18 @@ {% block stylesheets %} - - - - + + + + {% endblock %} {% block scripts %} {% if not nojs -%} - - - + + + {% endif %} {% endblock %} diff --git a/views/auth.njk b/views/auth.njk index 4b75c8f..26e935d 100644 --- a/views/auth.njk +++ b/views/auth.njk @@ -5,15 +5,15 @@ {% block stylesheets %} {{ super() }} - - + + {% endblock %} {% block scripts %} {{ super() }} - - - + + + {% endblock %} {% block content %} diff --git a/views/dashboard.njk b/views/dashboard.njk index f4ae437..b361bbf 100644 --- a/views/dashboard.njk +++ b/views/dashboard.njk @@ -5,22 +5,22 @@ {% block stylesheets %} {{ super() }} - - - - + + + + {% endblock %} {% block scripts %} {{ super() }} - - - - + + + + {# Polyfill smooth scroll for older browsers #} - - - + + + {% endblock %} {% block content %} @@ -89,7 +89,7 @@
- logo + logo
diff --git a/views/home.njk b/views/home.njk index 16a8184..8f64670 100644 --- a/views/home.njk +++ b/views/home.njk @@ -17,22 +17,22 @@ {% block stylesheets %} {{ super() }} - - - + + + {% endblock %} {% block scripts %} {{ super() }} - - - - - - - + + + + + + + {# We assign an ID for this so that the script can find out version string for render images #} - + {% endblock %} {% block content %} @@ -41,7 +41,7 @@

- +

{{ globals.name }}

{{ globals.home_subtitle | safe }}

diff --git a/views/nojs.njk b/views/nojs.njk index 71c2733..4694c79 100644 --- a/views/nojs.njk +++ b/views/nojs.njk @@ -16,7 +16,7 @@ {% block stylesheets %} {{ super() }} - + {% endblock %} {% block content %} @@ -25,7 +25,7 @@

- +

{{ globals.name }}

{{ globals.home_subtitle | safe }}