From 346d9864f38f5a6987063a3993a105c1a601fc88 Mon Sep 17 00:00:00 2001 From: Bobby Wibowo Date: Fri, 25 Dec 2020 21:06:21 +0700 Subject: [PATCH] improved codes for statistics much more expandable, and should be easier to understand overall. make more statistics operations run concurrently to speed them up. make linuxDiskStats config key obsolete by using systeminformation package to also query for any mounted file systems. --- config.sample.js | 8 - controllers/utilsController.js | 590 ++++++++++++++------------------- src/css/style.scss | 6 +- src/js/dashboard.js | 42 ++- 4 files changed, 274 insertions(+), 372 deletions(-) diff --git a/config.sample.js b/config.sample.js index b5c284b..22739e9 100644 --- a/config.sample.js +++ b/config.sample.js @@ -581,14 +581,6 @@ module.exports = { */ cacheControl: false, - /* - Enable Linux-only extended disk stats in Dashboard's Statistics. - This will use "du" binary to query disk usage of each directories within uploads directory. - Disabled by default as I personally found it to be very slow with +100k uploads - with my ancient potato server. - */ - linuxDiskStats: false, - /* Folder where to store logs. NOTE: This is currently unused. diff --git a/controllers/utilsController.js b/controllers/utilsController.js index ef8edaa..5a0837f 100644 --- a/controllers/utilsController.js +++ b/controllers/utilsController.js @@ -1,5 +1,4 @@ const { promisify } = require('util') -const { spawn } = require('child_process') const fetch = require('node-fetch') const ffmpeg = require('fluent-ffmpeg') const path = require('path') @@ -38,28 +37,33 @@ const self = { timezoneOffset: new Date().getTimezoneOffset() } -const statsCache = { +const statsData = { system: { + title: 'System', cache: null, generating: false, generatedAt: 0 }, - disk: { - cache: null, - generating: false, - generatedAt: 0 - }, - albums: { - cache: null, - generating: false, - generatedAt: 0 - }, - users: { + fileSystems: { + title: 'File Systems', cache: null, generating: false, generatedAt: 0 }, uploads: { + title: 'Uploads', + cache: null, + generating: false, + generatedAt: 0 + }, + users: { + title: 'Users', + cache: null, + generating: false, + generatedAt: 0 + }, + albums: { + title: 'Albums', cache: null, generating: false, generatedAt: 0 @@ -602,7 +606,7 @@ self.invalidateAlbumsCache = albumids => { self.invalidateStatsCache = type => { if (!['albums', 'users', 'uploads'].includes(type)) return - statsCache[type].cache = null + statsData[type].cache = null } self.stats = async (req, res, next) => { @@ -615,366 +619,266 @@ self.stats = async (req, res, next) => { try { const hrstart = process.hrtime() const stats = {} + Object.keys(statsData).forEach(key => { + // Pre-assign object keys to fix their display order + stats[statsData[key].title] = {} + }) + const os = await si.osInfo() + await Promise.all([ + (async () => { + // System info + const data = statsData.system - // System info - if (!statsCache.system.cache && statsCache.system.generating) { - stats.system = false - } else if (((Date.now() - statsCache.system.generatedAt) <= 1000) || statsCache.system.generating) { - // Use cache for 1000 ms (1 second) - stats.system = statsCache.system.cache - } else { - statsCache.system.generating = true - statsCache.system.generatedAt = Date.now() + if (!data.cache && data.generating) { + stats[data.title] = false + } else if (((Date.now() - data.generatedAt) <= 1000) || data.generating) { + // Use cache for 1000 ms (1 second) + stats[data.title] = data.cache + } else { + data.generating = true + data.generatedAt = Date.now() - const currentLoad = await si.currentLoad() - const mem = await si.mem() - const time = si.time() - const nodeUptime = process.uptime() + const currentLoad = await si.currentLoad() + const mem = await si.mem() + const time = si.time() + const nodeUptime = process.uptime() - if (self.clamscan.instance) { - try { - self.clamscan.version = await self.clamscan.instance.get_version().then(s => s.trim()) - } catch (error) { - logger.error(error) - self.clamscan.version = 'Errored when querying version.' + if (self.clamscan.instance) { + try { + self.clamscan.version = await self.clamscan.instance.get_version().then(s => s.trim()) + } catch (error) { + logger.error(error) + self.clamscan.version = 'Errored when querying version.' + } + } + + stats[data.title] = { + Platform: `${os.platform} ${os.arch}`, + Distro: `${os.distro} ${os.release}`, + Kernel: os.kernel, + Scanner: self.clamscan.version || 'N/A', + 'CPU Load': `${currentLoad.currentload.toFixed(1)}%`, + 'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '), + 'System Memory': { + value: { + used: mem.active, + total: mem.total + }, + type: 'byteUsage' + }, + 'Memory Usage': { + value: process.memoryUsage().rss, + type: 'byte' + }, + 'System Uptime': { + value: time.uptime, + type: 'uptime' + }, + 'Node.js': `${process.versions.node}`, + 'Service Uptime': { + value: Math.floor(nodeUptime), + type: 'uptime' + } + } + + // Update cache + data.cache = stats[data.title] + data.generating = false } - } + })(), + (async () => { + // File systems + const data = statsData.fileSystems - stats.system = { - _types: { - byte: ['memoryUsage'], - byteUsage: ['systemMemory'], - uptime: ['systemUptime', 'nodeUptime'] - }, - platform: `${os.platform} ${os.arch}`, - distro: `${os.distro} ${os.release}`, - kernel: os.kernel, - scanner: self.clamscan.version || 'N/A', - cpuLoad: `${currentLoad.currentload.toFixed(1)}%`, - cpusLoad: currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '), - systemMemory: { - used: mem.active, - total: mem.total - }, - memoryUsage: process.memoryUsage().rss, - systemUptime: time.uptime, - nodeVersion: `${process.versions.node}`, - nodeUptime: Math.floor(nodeUptime) - } + if (!data.cache && data.generating) { + stats[data.title] = false + } else if (((Date.now() - data.generatedAt) <= 60000) || data.generating) { + // Use cache for 60000 ms (60 seconds) + stats[data.title] = data.cache + } else { + data.generating = true + data.generatedAt = Date.now() - // Update cache - statsCache.system.cache = stats.system - statsCache.system.generating = false - } + stats[data.title] = {} - // Disk usage, only for Linux platform - if (os.platform === 'linux') { - if (!statsCache.disk.cache && statsCache.disk.generating) { - stats.disk = false - } else if (((Date.now() - statsCache.disk.generatedAt) <= 60000) || statsCache.disk.generating) { - // Use cache for 60000 ms (60 seconds) - stats.disk = statsCache.disk.cache - } else { - statsCache.disk.generating = true - statsCache.disk.generatedAt = Date.now() + const fsSize = await si.fsSize() + for (const fs of fsSize) { + stats[data.title][`${fs.fs} (${fs.type}) on ${fs.mount}`] = { + value: { + total: fs.size, + used: fs.used + }, + type: 'byteUsage' + } + } - stats.disk = { - _types: { - byteUsage: ['drive'] - }, - drive: null + // Update cache + data.cache = stats[data.title] + data.generating = false } + })(), + (async () => { + // Uploads + const data = statsData.uploads - // Linux-only extended disk stats - if (config.linuxDiskStats) { - // We pre-assign the keys below to fix their order - stats.disk._types.byte = ['uploads', 'thumbs', 'zips', 'chunks'] - stats.disk.uploads = 0 - stats.disk.thumbs = 0 - stats.disk.zips = 0 - stats.disk.chunks = 0 + if (!data.cache && data.generating) { + stats[data.title] = false + } else if (data.cache) { + // Cache will be invalidated with self.invalidateStatsCache() after any related operations + stats[data.title] = data.cache + } else { + data.generating = true + data.generatedAt = Date.now() - const subdirs = [] + stats[data.title] = { + Total: 0, + Images: 0, + Videos: 0, + Others: 0, + 'Size in DB': { + value: 0, + type: 'byte' + } + } - // Get size of uploads path (excluding sub-directories) - await new Promise((resolve, reject) => { - const proc = spawn('du', [ - '--apparent-size', - '--block-size=1', - '--dereference', - '--max-depth=1', - '--separate-dirs', - paths.uploads - ]) - - proc.stdout.on('data', data => { - const formatted = String(data) - .trim() - .split(/\s+/) - for (let i = 0; i < formatted.length; i += 2) { - const path = formatted[i + 1] - if (!path) return - - if (path !== paths.uploads) { - subdirs.push(path) - continue - } - - stats.disk.uploads = parseInt(formatted[i]) - } - }) - - const stderr = [] - proc.stderr.on('data', data => stderr.push(String(data))) - - proc.on('exit', code => { - if (code !== 0) return reject(stderr) - resolve() - }) - }) - - await Promise.all(subdirs.map(subdir => { - return new Promise((resolve, reject) => { - const proc = spawn('du', [ - '--apparent-size', - '--block-size=1', - '--dereference', - '--summarize', - subdir - ]) - - proc.stdout.on('data', data => { - const formatted = String(data) - .trim() - .split(/\s+/) - if (formatted.length !== 2) return - - const basename = path.basename(formatted[1]) - stats.disk[basename] = parseInt(formatted[0]) - - // Add to types if necessary - if (!stats.disk._types.byte.includes(basename)) { - stats.disk._types.byte.push(basename) - } - }) - - const stderr = [] - proc.stderr.on('data', data => stderr.push(String(data))) - - proc.on('exit', code => { - if (code !== 0) return reject(stderr) - resolve() - }) - }) - })) - } - - // Get disk usage of whichever disk uploads path resides on - await new Promise((resolve, reject) => { - const proc = spawn('df', [ - '--block-size=1', - '--output=size,avail', - paths.uploads + await Promise.all([ + (async () => { + const uploads = await db.table('files') + .select('size') + stats[data.title].Total = uploads.length + stats[data.title]['Size in DB'].value = uploads.reduce((acc, upload) => acc + parseInt(upload.size), 0) + })(), + (async () => { + stats[data.title].Images = await db.table('files') + .where(function () { + for (const ext of self.imageExts) { + this.orWhere('name', 'like', `%${ext}`) + } + }) + .count('id as count') + .then(rows => rows[0].count) + })(), + (async () => { + stats[data.title].Videos = await db.table('files') + .where(function () { + for (const ext of self.videoExts) { + this.orWhere('name', 'like', `%${ext}`) + } + }) + .count('id as count') + .then(rows => rows[0].count) + })() ]) - proc.stdout.on('data', data => { - // Only use the first valid line - if (stats.disk.drive !== null) return + stats[data.title].Others = stats[data.title].Total - stats[data.title].Images - stats[data.title].Videos - const lines = String(data) - .trim() - .split('\n') - if (lines.length !== 2) return + // Update cache + data.cache = stats[data.title] + data.generating = false + } + })(), + (async () => { + // Users + const data = statsData.users - for (const line of lines) { - const columns = line.split(/\s+/) - // Skip lines that have non-number chars - if (columns.some(w => !/^\d+$/.test(w))) continue + if (!data.cache && data.generating) { + stats[data.title] = false + } else if (data.cache) { + // Cache will be invalidated with self.invalidateStatsCache() after any related operations + stats[data.title] = data.cache + } else { + data.generating = true + data.generatedAt = Date.now() - const total = parseInt(columns[0]) - const avail = parseInt(columns[1]) - stats.disk.drive = { - total, - used: total - avail + stats[data.title] = { + Total: 0, + Disabled: 0 + } + + const permissionKeys = Object.keys(perms.permissions).reverse() + permissionKeys.forEach(p => { + stats[data.title][p] = 0 + }) + + const users = await db.table('users') + stats[data.title].Total = users.length + for (const user of users) { + if (user.enabled === false || user.enabled === 0) { + stats[data.title].Disabled++ + } + + user.permission = user.permission || 0 + for (const p of permissionKeys) { + if (user.permission === perms.permissions[p]) { + stats[data.title][p]++ + break } } - }) - - const stderr = [] - proc.stderr.on('data', data => stderr.push(String(data))) - - proc.on('exit', code => { - if (code !== 0) return reject(stderr) - resolve() - }) - }) - - // Update cache - statsCache.disk.cache = stats.disk - statsCache.disk.generating = false - } - } - - // Uploads - if (!statsCache.uploads.cache && statsCache.uploads.generating) { - stats.uploads = false - } else if (statsCache.uploads.cache) { - stats.uploads = statsCache.uploads.cache - } else { - statsCache.uploads.generating = true - statsCache.uploads.generatedAt = Date.now() - - stats.uploads = { - _types: { - number: ['total', 'images', 'videos', 'others'] - }, - total: 0, - images: 0, - videos: 0, - others: 0 - } - - if (!config.linuxDiskStats || os.platform !== 'linux') { - const uploads = await db.table('files') - .select('size') - stats.uploads.total = uploads.length - stats.uploads.sizeInDb = uploads.reduce((acc, upload) => acc + parseInt(upload.size), 0) - // Add type information for the new column - if (!Array.isArray(stats.uploads._types.byte)) { - stats.uploads._types.byte = [] - } - stats.uploads._types.byte.push('sizeInDb') - } else { - stats.uploads.total = await db.table('files') - .count('id as count') - .then(rows => rows[0].count) - } - - stats.uploads.images = await db.table('files') - .where(function () { - for (const ext of self.imageExts) { - this.orWhere('name', 'like', `%${ext}`) } - }) - .count('id as count') - .then(rows => rows[0].count) - stats.uploads.videos = await db.table('files') - .where(function () { - for (const ext of self.videoExts) { - this.orWhere('name', 'like', `%${ext}`) + // Update cache + data.cache = stats[data.title] + data.generating = false + } + })(), + (async () => { + // Albums + const data = statsData.albums + + if (!data.cache && data.generating) { + stats[data.title] = false + } else if (data.cache) { + // Cache will be invalidated with self.invalidateStatsCache() after any related operations + stats[data.title] = data.cache + } else { + data.generating = true + data.generatedAt = Date.now() + + stats[data.title] = { + Total: 0, + Disabled: 0, + Public: 0, + Downloadable: 0, + 'ZIP Generated': 0 } - }) - .count('id as count') - .then(rows => rows[0].count) - stats.uploads.others = stats.uploads.total - stats.uploads.images - stats.uploads.videos - - // Update cache - statsCache.uploads.cache = stats.uploads - statsCache.uploads.generating = false - } - - // Users - if (!statsCache.users.cache && statsCache.users.generating) { - stats.users = false - } else if (statsCache.users.cache) { - stats.users = statsCache.users.cache - } else { - statsCache.users.generating = true - statsCache.users.generatedAt = Date.now() - - stats.users = { - _types: { - number: ['total', 'disabled'] - }, - total: 0, - disabled: 0 - } - - const permissionKeys = Object.keys(perms.permissions).reverse() - permissionKeys.forEach(p => { - stats.users[p] = 0 - stats.users._types.number.push(p) - }) - - const users = await db.table('users') - stats.users.total = users.length - for (const user of users) { - if (user.enabled === false || user.enabled === 0) { - stats.users.disabled++ - } - - // This may be inaccurate on installations with customized permissions - user.permission = user.permission || 0 - for (const p of permissionKeys) { - if (user.permission === perms.permissions[p]) { - stats.users[p]++ - break + const albums = await db.table('albums') + stats[data.title].Total = albums.length + const identifiers = [] + for (const album of albums) { + if (!album.enabled) { + stats[data.title].Disabled++ + continue + } + if (album.download) stats[data.title].Downloadable++ + if (album.public) stats[data.title].Public++ + if (album.zipGeneratedAt) identifiers.push(album.identifier) } + + await Promise.all(identifiers.map(async identifier => { + try { + await paths.access(path.join(paths.zips, `${identifier}.zip`)) + stats[data.title]['ZIP Generated']++ + } catch (error) { + // Re-throw error + if (error.code !== 'ENOENT') throw error + } + })) + + // Update cache + data.cache = stats[data.title] + data.generating = false } - } - - // Update cache - statsCache.users.cache = stats.users - statsCache.users.generating = false - } - - // Albums - if (!statsCache.albums.cache && statsCache.albums.generating) { - stats.albums = false - } else if (statsCache.albums.cache) { - stats.albums = statsCache.albums.cache - } else { - statsCache.albums.generating = true - statsCache.albums.generatedAt = Date.now() - - stats.albums = { - _types: { - number: ['total', 'active', 'downloadable', 'public', 'generatedZip'] - }, - total: 0, - disabled: 0, - public: 0, - downloadable: 0, - zipGenerated: 0 - } - - const albums = await db.table('albums') - stats.albums.total = albums.length - const identifiers = [] - for (const album of albums) { - if (!album.enabled) { - stats.albums.disabled++ - continue - } - if (album.download) stats.albums.downloadable++ - if (album.public) stats.albums.public++ - if (album.zipGeneratedAt) identifiers.push(album.identifier) - } - - await Promise.all(identifiers.map(async identifier => { - try { - await paths.access(path.join(paths.zips, `${identifier}.zip`)) - stats.albums.zipGenerated++ - } catch (error) { - // Re-throw error - if (error.code !== 'ENOENT') throw error - } - })) - - // Update cache - statsCache.albums.cache = stats.albums - statsCache.albums.generating = false - } + })() + ]) return res.json({ success: true, stats, hrtime: process.hrtime(hrstart) }) } catch (error) { logger.error(error) // Reset generating state when encountering any errors - Object.keys(statsCache).forEach(key => { - statsCache[key].generating = false + Object.keys(statsData).forEach(key => { + statsData[key].generating = false }) return res.status(500).json({ success: false, description: 'An unexpected error occurred. Try again?' }) } diff --git a/src/css/style.scss b/src/css/style.scss index f7f5ad3..9b8bb6f 100644 --- a/src/css/style.scss +++ b/src/css/style.scss @@ -259,11 +259,7 @@ fieldset[disabled] .select select:hover { th { color: $white-ter; height: 2.25em; - font-weight: normal; - - &.capitalize { - text-transform: capitalize - } + font-weight: normal } thead { diff --git a/src/js/dashboard.js b/src/js/dashboard.js index d714256..d3cc010 100644 --- a/src/js/dashboard.js +++ b/src/js/dashboard.js @@ -2871,29 +2871,39 @@ page.getStatistics = (params = {}) => { ` } else { try { - const types = response.data.stats[keys[i]]._types || {} const valKeys = Object.keys(response.data.stats[keys[i]]) for (let j = 0; j < valKeys.length; j++) { - // Skip keys that starts with an underscore - if (/^_/.test(valKeys[j])) continue + const data = response.data.stats[keys[i]][valKeys[j]] + const type = typeof data === 'object' ? data.type : 'auto' + const value = typeof data === 'object' ? data.value : data - const value = response.data.stats[keys[i]][valKeys[j]] - let parsed = value - - // Parse values with some preset formatting - if ((types.number || []).includes(valKeys[j])) parsed = value.toLocaleString() - if ((types.byte || []).includes(valKeys[j])) parsed = page.getPrettyBytes(value) - if ((types.byteUsage || []).includes(valKeys[j])) { - parsed = `${page.getPrettyBytes(value.used)} / ${page.getPrettyBytes(value.total)} (${Math.floor(value.used / value.total * 100)}%)` + let parsed + switch (type) { + case 'byte': + parsed = page.getPrettyBytes(value) + break + case 'byteUsage': + parsed = `${page.getPrettyBytes(value.used)} / ${page.getPrettyBytes(value.total)} (${Math.floor(value.used / value.total * 100)}%)` + break + case 'uptime': + parsed = page.getPrettyUptime(value) + break + case 'auto': + switch (typeof value) { + case 'number': + parsed = value.toLocaleString() + break + default: + parsed = value + } + break + default: + parsed = value } - if ((types.uptime || []).includes(valKeys[j])) parsed = page.getPrettyUptime(value) - const string = valKeys[j] - .replace(/([A-Z])/g, ' $1') - .replace(/(^|\s)(cpu|db|zip)/gi, s => s.toUpperCase()) rows += ` - ${string} + ${valKeys[j]} ${parsed} `