Re-worked caching for statistics

I guess I'll work on adding charts someday.

controllers/albumsController.js

@@ -101,6 +101,7 @@ albumsController.create = async (req, res, next) => {
     public: (req.body.public === false || req.body.public === 0) ? 0 : 1,
     description: utils.escape(req.body.description) || ''
   })
+  utils.invalidateStatsCache('albums')
 
   return res.json({ success: true, id: ids[0] })
 }
@@ -156,6 +157,7 @@ albumsController.delete = async (req, res, next) => {
       userid: user.id
     })
     .update('enabled', 0)
+  utils.invalidateStatsCache('albums')
 
   const identifier = await db.table('albums')
     .select('identifier')
@@ -216,6 +218,7 @@ albumsController.edit = async (req, res, next) => {
       public: Boolean(req.body.public),
       description: utils.escape(req.body.description) || ''
     })
+  utils.invalidateStatsCache('albums')
 
   if (req.body.requestLink) {
     const oldIdentifier = await db.table('albums')
@@ -416,6 +419,7 @@ albumsController.generateZip = async (req, res, next) => {
             const fileName = `${album.name}.zip`
 
             albumsController.zipEmitters.get(identifier).emit('done', filePath, fileName)
+            utils.invalidateStatsCache('albums')
             return download(filePath, fileName)
           })
     })

controllers/authController.js

@@ -63,6 +63,7 @@ authController.register = async (req, res, next) => {
       enabled: 1,
       permission: perms.permissions.user
     })
+    utils.invalidateStatsCache('users')
     return res.json({ success: true, token })
   })
 }
@@ -179,6 +180,7 @@ authController.editUser = async (req, res, next) => {
   await db.table('users')
     .where('id', id)
     .update(update)
+  utils.invalidateStatsCache('users')
 
   if (!req.body.resetPassword)
     return res.json({ success: true, update })

controllers/uploadController.js

@@ -550,6 +550,7 @@ uploadsController.formatInfoMap = (req, res, user, infoMap) => {
             userid: user !== undefined ? user.id : null,
             timestamp: Math.floor(Date.now() / 1000)
           })
+          utils.invalidateStatsCache('uploads')
         } else {
           utils.deleteFile(info.data.filename, req.app.get('uploads-set')).catch(console.error)
           existingFiles.push(dbFile)
@@ -669,6 +670,7 @@ uploadsController.bulkDelete = async (req, res) => {
     return res.json({ success: false, description: 'No array of files specified.' })
 
   const failed = await utils.bulkDeleteFiles(field, values, user, req.app.get('uploads-set'))
+  utils.invalidateStatsCache('uploads')
   if (failed.length < values.length)
     return res.json({ success: true, failed })
 

controllers/utilsController.js

@@ -10,6 +10,24 @@ const perms = require('./permissionController')
 const sharp = require('sharp')
 
 const utilsController = {}
+const _stats = {
+  system: {
+    cache: null,
+    timestamp: 0
+  },
+  albums: {
+    cache: null,
+    valid: false
+  },
+  users: {
+    cache: null,
+    valid: false
+  },
+  uploads: {
+    cache: null,
+    valid: false
+  }
+}
 
 const uploadsDir = path.join(__dirname, '..', config.uploads.folder)
 const thumbsDir = path.join(uploadsDir, 'thumbs')
@@ -405,7 +423,8 @@ utilsController.purgeCloudflareCache = async (names, uploads, thumbs) => {
   return results
 }
 
-utilsController.getLinuxMemoryUsage = () => {
+utilsController.getMemoryUsage = () => {
+  // For now this is linux-only. Not sure if darwin has this too.
   return new Promise((resolve, reject) => {
     const prc = spawn('free', ['-b'])
     prc.stdout.setEncoding('utf8')
@@ -432,6 +451,12 @@ utilsController.getLinuxMemoryUsage = () => {
   })
 }
 
+utilsController.invalidateStatsCache = type => {
+  if (!['albums', 'users', 'uploads'].includes(type)) return
+  _stats[type].cache = null
+  _stats[type].valid = false
+}
+
 utilsController.stats = async (req, res, next) => {
   const user = await utilsController.authorize(req, res)
   if (!user) return
@@ -439,66 +464,149 @@ utilsController.stats = async (req, res, next) => {
   const isadmin = perms.is(user, 'admin')
   if (!isadmin) return res.status(403).end()
 
-  const platform = os.platform()
+  const stats = {}
-  const system = { platform: `${platform}-${os.arch()}` }
+
-  if (platform === 'linux') {
+  // Re-use system cache for only 1000ms
-    const memoryUsage = await utilsController.getLinuxMemoryUsage()
+  if (Date.now() - _stats.system.timestamp <= 1000) {
-    system.memory = {
+    stats.system = _stats.system.cache
-      used: memoryUsage.mem.used,
+  } else {
-      total: memoryUsage.mem.total
+    const platform = os.platform()
+    stats.system = {
+      platform: `${platform}-${os.arch()}`,
+      systemMemory: null,
+      nodeVersion: `${process.versions.node}`,
+      memoryUsage: process.memoryUsage().rss
     }
-  }
 
-  system['node.js'] = `${process.versions.node}`
+    if (platform === 'linux') {
-  system['memory usage'] = process.memoryUsage().rss
+      const memoryUsage = await utilsController.getMemoryUsage()
-
+      stats.system.systemMemory = {
-  if (platform !== 'win32')
+        used: memoryUsage.mem.used,
-    system.loadavg = `${os.loadavg().map(load => load.toFixed(2)).join(', ')}`
+        total: memoryUsage.mem.total
-
-  const stats = {
-    uploads: {
-      count: 0,
-      size: 0,
-      types: {
-        images: 0,
-        videos: 0,
-        others: 0
       }
-    },
+    } else {
-    users: {
+      delete stats.system.systemMemory
-      count: 0,
+    }
-      disabled: 0,
+
-      permissions: {}
+    if (platform !== 'win32')
+      stats.system.loadAverage = `${os.loadavg().map(load => load.toFixed(2)).join(', ')}`
+
+    // Cache
+    _stats.system = {
+      cache: stats.system,
+      timestamp: Date.now()
     }
   }
 
-  Object.keys(perms.permissions).forEach(p => {
+  // Re-use albums, users, and uploads caches as long as they are still valid
-    stats.users.permissions[p] = 0
-  })
 
-  const uploads = await db.table('files')
+  if (_stats.albums.valid) {
-  stats.uploads.count = uploads.length
+    stats.albums = _stats.albums.cache
-  for (const upload of uploads) {
+  } else {
-    stats.uploads.size += parseInt(upload.size)
+    stats.albums = {
-    const extname = utilsController.extname(upload.name)
+      total: 0,
-    if (utilsController.imageExtensions.includes(extname))
+      active: 0,
-      stats.uploads.types.images++
+      downloadable: 0,
-    else if (utilsController.videoExtensions.includes(extname))
+      public: 0,
-      stats.uploads.types.videos++
+      zips: 0
-    else
+    }
-      stats.uploads.types.others++
+
+    const albums = await db.table('albums')
+    stats.albums.total = albums.length
+    const identifiers = []
+    for (const album of albums)
+      if (album.enabled) {
+        stats.albums.active++
+        if (album.download) stats.albums.downloadable++
+        if (album.public) stats.albums.public++
+        if (album.zipGeneratedAt) identifiers.push(album.identifier)
+      }
+
+    const zipsDir = path.join(uploadsDir, 'zips')
+    await Promise.all(identifiers.map(identifier => {
+      return new Promise(resolve => {
+        const filePath = path.join(zipsDir, `${identifier}.zip`)
+        fs.access(filePath, error => {
+          if (!error) stats.albums.zips++
+          resolve(true)
+        })
+      })
+    }))
+
+    // Cache
+    _stats.albums = {
+      cache: stats.albums,
+      valid: true
+    }
   }
 
-  const users = await db.table('users')
+  if (_stats.users.valid) {
-  stats.users.count = users.length
+    stats.users = _stats.users.cache
-  for (const user of users) {
+  } else {
-    if (user.enabled === false || user.enabled === 0) stats.users.disabled++
+    stats.users = {
-    user.permission = user.permission || 0
+      total: 0,
-    for (const p of Object.keys(stats.users.permissions))
+      disabled: 0
-      if (user.permission === perms.permissions[p]) stats.users.permissions[p]++
+    }
+
+    const permissionKeys = Object.keys(perms.permissions)
+    permissionKeys.forEach(p => {
+      stats.users[p] = 0
+    })
+
+    const users = await db.table('users')
+    stats.users.total = users.length
+    for (const user of users) {
+      if (user.enabled === false || user.enabled === 0)
+        stats.users.disabled++
+
+      // This may be inaccurate on installations with customized permissions
+      user.permission = user.permission || 0
+      for (const p of permissionKeys)
+        if (user.permission === perms.permissions[p]) {
+          stats.users[p]++
+          break
+        }
+    }
+
+    // Cache
+    _stats.users = {
+      cache: stats.users,
+      valid: true
+    }
   }
 
-  return res.json({ success: true, system, stats })
+  if (_stats.uploads.valid) {
+    stats.uploads = _stats.uploads.cache
+  } else {
+    stats.uploads = {
+      total: 0,
+      size: 0,
+      images: 0,
+      videos: 0,
+      others: 0
+    }
+
+    const uploads = await db.table('files')
+    stats.uploads.total = uploads.length
+    for (const upload of uploads) {
+      stats.uploads.size += parseInt(upload.size)
+      const extname = utilsController.extname(upload.name)
+      if (utilsController.imageExtensions.includes(extname))
+        stats.uploads.images++
+      else if (utilsController.videoExtensions.includes(extname))
+        stats.uploads.videos++
+      else
+        stats.uploads.others++
+    }
+
+    // Cache
+    _stats.uploads = {
+      cache: stats.uploads,
+      valid: true
+    }
+  }
+
+  return res.json({ success: true, stats })
 }
 
 module.exports = utilsController

public/js/dashboard.js

@@ -1988,77 +1988,45 @@ page.getServerStats = function (element) {
         return swal('An error occurred!', response.data.description, 'error')
       }
 
-    const system = response.data.system
+    let content = ''
-    let systemRows = ''
+
-    for (const s of Object.keys(system)) {
+    for (const key of Object.keys(response.data.stats)) {
-      let value
+      let rows = ''
-      if (s === 'memory') {
+      for (const valKey of Object.keys(response.data.stats[key])) {
-        const mem = system[s]
+        let value = response.data.stats[key][valKey]
-        value = `${page.getPrettyBytes(mem.used)} / ${page.getPrettyBytes(mem.total)} (${Math.round((mem.used / mem.total) * 100)}%)`
+        if (['albums', 'users'].includes(key))
-      } else if (s === 'memory usage') {
+          value = value.toLocaleString()
-        value = page.getPrettyBytes(system[s])
+        if (['memoryUsage', 'size'].includes(valKey))
-      } else {
+          value = page.getPrettyBytes(value)
-        value = system[s].toLocaleString()
+        if (valKey === 'systemMemory')
+          value = `${page.getPrettyBytes(value.used)} / ${page.getPrettyBytes(value.total)} (${Math.round(value.used / value.total * 100)}%)`
+        rows += `
+          <tr>
+            <th>${valKey.replace(/([A-Z])/g, ' $1').toUpperCase()}</th>
+            <td>${value}</td>
+          </tr>
+        `
       }
-      systemRows += `
+      content += `
-        <tr>
+        <div class="table-container">
-          <th>${s.toUpperCase()}</th>
+          <table class="table is-fullwidth is-hoverable">
-          <td>${value}</td>
+            <thead>
-        </tr>
+              <tr>
+                <th>${key.toUpperCase()}</th>
+                <td style="width: 50%"></td>
+              </tr>
+            </thead>
+            <tbody>
+              ${rows}
+            </tbody>
+          </table>
+        </div>
       `
     }
 
-    const types = response.data.stats.uploads.types
-    let typesRows = ''
-    for (const t of Object.keys(types))
-      typesRows += `
-        <tr>
-          <th class="cell-indent">${t.toUpperCase()}</th>
-          <td>${types[t].toLocaleString()}</td>
-        </tr>
-      `
-
-    const permissions = response.data.stats.users.permissions
-    let permissionsRows = ''
-    for (const p of Object.keys(permissions))
-      permissionsRows += `
-        <tr>
-          <th class="cell-indent">${p.toUpperCase()}</th>
-          <td>${permissions[p].toLocaleString()}</td>
-        </tr>
-      `
-
     page.dom.innerHTML = `
       <h2 class="subtitle">Statistics</h2>
-      <div class="table-container">
+      ${content}
-        <table class="table is-fullwidth is-hoverable">
-          <tbody>
-            ${systemRows}
-            <tr>
-              <th>DISK USAGE</th>
-              <td>${page.getPrettyBytes(response.data.stats.uploads.size)}</td>
-            </tr>
-            <tr>
-              <th class="cell-indent">IN BYTES</th>
-              <td>${response.data.stats.uploads.size.toLocaleString()} B</td>
-            </tr>
-            <tr>
-              <th>TOTAL UPLOADS</th>
-              <td>${response.data.stats.uploads.count.toLocaleString()}</td>
-            </tr>
-            ${typesRows}
-            <tr>
-              <th>TOTAL USERS</th>
-              <td>${response.data.stats.users.count.toLocaleString()}</td>
-            </tr>
-            ${permissionsRows}
-            <tr>
-              <th class="cell-indent">DISABLED</th>
-              <td>${response.data.stats.users.disabled.toLocaleString()}</td>
-            </tr>
-          </tbody>
-        </table>
-      </div>
     `
 
     page.fadeIn()

views/_globals.njk

@@ -16,7 +16,7 @@
   v3: CSS and JS files (libs such as bulma, lazyload, etc).
   v4: Renders in /public/render/* directories (to be used by render.js).
 #}
-{% set v1 = "xz10E2pAFd" %}
+{% set v1 = "MYXYdOtS8u" %}
 {% set v2 = "hiboQUzAzp" %}
 {% set v3 = "hiboQUzAzp" %}
 {% set v4 = "dplQUZqTnf" %}