feat: custom livedirectory middleware

with conditional gets support

controllers/middlewares/expressCompat.js

@@ -0,0 +1,31 @@
+// TODO: Currently consulting with author of hyper-express
+// about the behavior of Response.get()/.getHeader() not matching ExpressJS
+// https://github.com/kartikk221/hyper-express/discussions/97
+// This middleware is a workaround, hopefully only temporarily
+
+class ExpressCompat {
+  #getHeader (res, name) {
+    // Always return first value in array if it only has a single value
+    const values = res._getHeader(name)
+    if (Array.isArray(values) && values.length === 1) {
+      return values[0]
+    } else {
+      return values
+    }
+  }
+
+  #middleware (req, res, next) {
+    // Alias Response.get() and Response.getHeader() with a function that is more aligned with ExpressJS
+    res._get = res.get
+    res._getHeader = res.getHeader
+    res.get = res.getHeader = name => this.#getHeader(res, name)
+
+    return next()
+  }
+
+  get middleware () {
+    return this.#middleware.bind(this)
+  }
+}
+
+module.exports = ExpressCompat

controllers/middlewares/serveLiveDirectory.js

@@ -0,0 +1,85 @@
+const LiveDirectory = require('live-directory')
+const serveUtils = require('../utils/serveUtils')
+
+class ServeLiveDirectory {
+  instance
+
+  #options
+
+  constructor (instanceOptions = {}, options = {}) {
+    if (!instanceOptions.ignore) {
+      instanceOptions.ignore = path => {
+        // ignore dot files
+        return path.startsWith('.')
+      }
+    }
+
+    this.instance = new LiveDirectory(instanceOptions)
+
+    if (options.setHeaders && typeof options.setHeaders !== 'function') {
+      throw new TypeError('Middleware option setHeaders must be a function')
+    }
+
+    this.#options = options
+  }
+
+  #middleware (req, res, next) {
+    // Only process GET and HEAD requests
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+      return next()
+    }
+
+    const file = this.instance.get(req.path)
+    if (file === undefined) {
+      return next()
+    }
+
+    // set header fields
+    this.#setHeaders(req, res, file)
+
+    // set content-type
+    res.type(file.extension)
+
+    // conditional GET support
+    if (serveUtils.isConditionalGET(req)) {
+      if (serveUtils.isPreconditionFailure(req, res)) {
+        return res.status(412).end()
+      }
+
+      if (serveUtils.isFresh(req, res)) {
+        return res.status(304).end()
+      }
+    }
+
+    // HEAD support
+    if (req.method === 'HEAD') {
+      return res.end()
+    }
+
+    return res.send(file.buffer)
+  }
+
+  #setHeaders (req, res, file) {
+    // Always do external setHeaders function first,
+    // in case it will overwrite the following default headers anyways
+    if (this.#options.setHeaders) {
+      this.#options.setHeaders(req, res)
+    }
+
+    if (!res.get('Last-Modified')) {
+      const modified = new Date(file.last_update).toUTCString()
+      res.header('Last-Modified', modified)
+    }
+
+    if (!res.get('ETag')) {
+      const val = file.etag
+      res.header('ETag', val)
+    }
+  }
+
+  get middleware () {
+    return this.#middleware.bind(this)
+  }
+}
+
+module.exports = ServeLiveDirectory

controllers/utils/serveUtils.js

@@ -0,0 +1,188 @@
+const self = {}
+
+/*
+* https://github.com/jshttp/fresh/blob/v0.5.2/index.js
+* Copyright(c) 2012 TJ Holowaychuk
+* Copyright(c) 2016-2017 Douglas Christopher Wilson
+* MIT Licensed
+*/
+
+const CACHE_CONTROL_NO_CACHE_REGEXP = /(?:^|,)\s*?no-cache\s*?(?:,|$)/
+
+self.fresh = (reqHeaders, resHeaders) => {
+  // fields
+  const modifiedSince = reqHeaders['if-modified-since']
+  const noneMatch = reqHeaders['if-none-match']
+
+  // unconditional request
+  if (!modifiedSince && !noneMatch) {
+    return false
+  }
+
+  // Always return stale when Cache-Control: no-cache
+  // to support end-to-end reload requests
+  // https://tools.ietf.org/html/rfc2616#section-14.9.4
+  const cacheControl = reqHeaders['cache-control']
+  if (cacheControl && CACHE_CONTROL_NO_CACHE_REGEXP.test(cacheControl)) {
+    return false
+  }
+
+  // if-none-match
+  if (noneMatch && noneMatch !== '*') {
+    const etag = resHeaders.etag
+
+    if (!etag) {
+      return false
+    }
+
+    let etagStale = true
+    const matches = self.parseTokenList(noneMatch)
+    for (let i = 0; i < matches.length; i++) {
+      const match = matches[i]
+      if (match === etag || match === 'W/' + etag || 'W/' + match === etag) {
+        etagStale = false
+        break
+      }
+    }
+
+    if (etagStale) {
+      return false
+    }
+  }
+
+  // if-modified-since
+  if (modifiedSince) {
+    const lastModified = resHeaders['last-modified']
+    const modifiedStale = !lastModified || !(self.parseHttpDate(lastModified) <= self.parseHttpDate(modifiedSince))
+
+    if (modifiedStale) {
+      return false
+    }
+  }
+
+  return true
+}
+
+self.isFresh = (req, res) => {
+  return self.fresh(req.headers, {
+    etag: res.get('ETag'),
+    'last-modified': res.get('Last-Modified')
+  })
+}
+
+/*
+ * https://github.com/pillarjs/send/blob/0.18.0/index.js
+ * Copyright(c) 2012 TJ Holowaychuk
+ * Copyright(c) 2014-2022 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+self.isRangeFresh = (req, res) => {
+  const ifRange = req.headers['if-range']
+
+  if (!ifRange) {
+    return true
+  }
+
+  // if-range as etag
+  if (ifRange.indexOf('"') !== -1) {
+    const etag = res.get('ETag')
+    return Boolean(etag && ifRange.indexOf(etag) !== -1)
+  }
+
+  // if-range as modified date
+  const lastModified = res.get('Last-Modified')
+  return self.parseHttpDate(lastModified) <= self.parseHttpDate(ifRange)
+}
+
+self.isConditionalGET = req => {
+  return req.headers['if-match'] ||
+    req.headers['if-unmodified-since'] ||
+    req.headers['if-none-match'] ||
+    req.headers['if-modified-since']
+}
+
+self.isPreconditionFailure = (req, res) => {
+  // if-match
+  const match = req.headers['if-match']
+  if (match) {
+    const etag = res.get('ETag')
+    return !etag || (match !== '*' && self.parseTokenList(match).every(match => {
+      return match !== etag && match !== 'W/' + etag && 'W/' + match !== etag
+    }))
+  }
+
+  // if-unmodified-since
+  const unmodifiedSince = self.parseHttpDate(req.headers['if-unmodified-since'])
+  if (!isNaN(unmodifiedSince)) {
+    const lastModified = self.parseHttpDate(res.get('Last-Modified'))
+    return isNaN(lastModified) || lastModified > unmodifiedSince
+  }
+
+  return false
+}
+
+/*
+// TODO: ServeStatic may need these, but ServeLiveDirectory does its own (since it does not need Accept-Ranges support)
+self.setHeader = (res, path, stat) => {
+  if (this._acceptRanges && !res.get('Accept-Ranges')) {
+    logger.debug('accept ranges')
+    res.header('Accept-Ranges', 'bytes')
+  }
+
+  if (this._lastModified && !res.get('Last-Modified')) {
+    const modified = stat.mtime.toUTCString()
+    logger.debug('modified %s', modified)
+    res.header('Last-Modified', modified)
+  }
+
+  if (this._etag && !res.get('ETag')) {
+    const val = etag(stat)
+    logger.debug('etag %s', val)
+    res.header('ETag', val)
+  }
+}
+*/
+
+self.parseHttpDate = date => {
+  const timestamp = date && Date.parse(date)
+
+  return typeof timestamp === 'number'
+    ? timestamp
+    : NaN
+}
+
+self.parseTokenList = str => {
+  let end = 0
+  const list = []
+  let start = 0
+
+  // gather tokens
+  for (let i = 0, len = str.length; i < len; i++) {
+    switch (str.charCodeAt(i)) {
+      case 0x20: /*   */
+        if (start === end) {
+          start = end = i + 1
+        }
+        break
+      case 0x2c: /* , */
+        if (start !== end) {
+          list.push(str.substring(start, end))
+        }
+        start = end = i + 1
+        break
+      default:
+        end = i + 1
+        break
+    }
+  }
+
+  // final token
+  if (start !== end) {
+    list.push(str.substring(start, end))
+  }
+
+  return list
+}
+
+module.exports = self

lolisafe.js

@@ -44,8 +44,10 @@ paths.initSync()
 const utils = require('./controllers/utilsController')
 
 // Custom middlewares
+const ExpressCompat = require('./controllers/middlewares/expressCompat')
 const NunjucksRenderer = require('./controllers/middlewares/nunjucksRenderer')
 const RateLimiter = require('./controllers/middlewares/rateLimiter')
+const ServeLiveDirectory = require('./controllers/middlewares/serveLiveDirectory')
 // const ServeStatic = require('./controllers/middlewares/serveStatic') // TODO
 
 // Routes
@@ -57,6 +59,10 @@ const player = require('./routes/player')
 
 const isDevMode = process.env.NODE_ENV === 'development'
 
+// Express-compat
+const expressCompatInstance = new ExpressCompat()
+safe.use(expressCompatInstance.middleware)
+
 // Rate limiters
 if (Array.isArray(config.rateLimiters)) {
   let whitelistedKeys
@@ -119,17 +125,9 @@ const nunjucksRendererInstance = new NunjucksRenderer('views', {
 })
 safe.use('/', nunjucksRendererInstance.middleware)
 
-const initLiveDirectory = (options = {}) => {
-  if (!options.ignore) {
-    options.ignore = path => {
-      // ignore dot files
-      return path.startsWith('.')
-    }
-  }
-  return new LiveDirectory(options)
-}
-
-const cdnPages = [...config.pages]
+// Array of routes to apply CDN Cache-Control onto,
+// and additionally call Cloudflare API to have their CDN caches purged when lolisafe starts
+const cdnRoutes = [...config.pages]
 
 // Defaults to no-op
 let setHeadersForStaticAssets = () => {}
@@ -156,12 +154,14 @@ if (config.cacheControl) {
   switch (config.cacheControl) {
     case 1:
     case true:
-      // If using CDN, cache public pages in CDN
-      cdnPages.push('api/check')
+      // If using CDN, cache most front-end pages in CDN
+      // Include /api/check since it will only reply with persistent JSON payload
+      // that will not change, unless config file is edited and lolisafe is then restarted
+      cdnRoutes.push('api/check')
       safe.use((req, res, next) => {
         if (req.method === 'GET' || req.method === 'HEAD') {
           const page = req.path === '/' ? 'home' : req.path.substring(1)
-          if (cdnPages.includes(page)) {
+          if (cdnRoutes.includes(page)) {
             res.header('Cache-Control', cacheControls.cdn)
           }
         }
@@ -190,23 +190,16 @@ if (config.cacheControl) {
 }
 
 // Static assets
-const liveDirectoryPublic = initLiveDirectory({ path: paths.public })
-const liveDirectoryDist = initLiveDirectory({ path: paths.dist })
-safe.use('/', (req, res, next) => {
-  // Only process GET and HEAD requests
-  if (req.method === 'GET' || req.method === 'HEAD') {
-    // Try to find asset from public directory, then dist directory
-    const file =
-      liveDirectoryPublic.get(req.path) ||
-      liveDirectoryDist.get(req.path)
-    if (file === undefined) {
-      return next()
-    }
-    setHeadersForStaticAssets(req, res)
-    return res.type(file.extension).send(file.buffer)
-  }
-  return next()
-})
+const serveLiveDirectoryPublicInstance = new ServeLiveDirectory(
+  { path: paths.public },
+  { setHeaders: setHeadersForStaticAssets }
+)
+safe.use('/', serveLiveDirectoryPublicInstance.middleware)
+const serveLiveDirectoryDistInstance = new ServeLiveDirectory(
+  { path: paths.dist },
+  { setHeaders: setHeadersForStaticAssets }
+)
+safe.use('/', serveLiveDirectoryDistInstance.middleware)
 
 // Routes
 safe.use('/', album)
@@ -239,9 +232,13 @@ safe.use('/api', api)
       }
     }
 
-    const liveDirectoryCustomPages = initLiveDirectory({
+    const liveDirectoryCustomPages = new LiveDirectory({
       path: paths.customPages,
-      keep: ['.html']
+      keep: ['.html'],
+      ignore: path => {
+        // ignore dot files
+        return path.startsWith('.')
+      }
     })
 
     // Cookie Policy
@@ -259,6 +256,7 @@ safe.use('/api', api)
         const page = req.path === '/' ? 'home' : req.path.substring(1)
         const customPage = liveDirectoryCustomPages.get(`${page}.html`)
         if (customPage) {
+          // TODO: Conditional GETs? (e.g. Last-Modified, etag, etc.)
           return res.type('html').send(customPage.buffer)
         } else if (config.pages.includes(page)) {
           // These rendered pages are persistently cached during production
@@ -332,7 +330,7 @@ safe.use('/api', api)
     if (config.cacheControl && config.cacheControl !== 2) {
       if (config.cloudflare.purgeCache) {
         logger.log('Cache control enabled, purging Cloudflare\'s cache...')
-        const results = await utils.purgeCloudflareCache(cdnPages)
+        const results = await utils.purgeCloudflareCache(cdnRoutes)
         let errored = false
         let succeeded = 0
         for (const result of results) {