feat: StatsManager

This commit is contained in:
Bobby 2022-10-06 02:46:49 +07:00
parent 364e16af7f
commit 0c0b2aa3d1
No known key found for this signature in database
GPG Key ID: 941839794CBF5A09
4 changed files with 410 additions and 405 deletions

View File

@ -0,0 +1,350 @@
const jetpack = require('fs-jetpack')
const si = require('systeminformation')
const paths = require('./../pathsController')
const perms = require('./../permissionController')
const Constants = require('./Constants')
const logger = require('./../../logger')
const Type = Object.freeze({
// Should contain key value: number
UPTIME: 'uptime',
// Should contain key value: number
BYTE: 'byte',
// Should contain key value: { used: number, total: number }
BYTE_USAGE: 'byteUsage',
// Should contain key value: number
TEMP_CELSIUS: 'tempC',
// Should contain key data: Array<{ key: string, value: number | string }>
// and optionally a count/total
DETAILED: 'detailed',
// Should contain key value: null
// May consider still displaying entries with this type in the frontend,
// but mark as unavailable explicitly due to backend lacking the capabilities
UNAVAILABLE: 'unavailable',
// Hidden type should be skipped during iteration, can contain anything
// These should be treated on a case by case basis on the frontend
HIDDEN: 'hidden'
})
const self = {
_buildExtsRegex: exts => {
const str = exts.map(ext => ext.substring(1)).join('|')
return new RegExp(`\\.(${str})$`, 'i')
},
cachedStats: {}
}
self.imageExtsRegex = self._buildExtsRegex(Constants.IMAGE_EXTS)
self.videoExtsRegex = self._buildExtsRegex(Constants.VIDEO_EXTS)
self.audioExtsRegex = self._buildExtsRegex(Constants.AUDIO_EXTS)
self.invalidateStatsCache = type => {
if (!self.cachedStats[type]) return
self.cachedStats[type].cache = null
}
self.getSystemInfo = async () => {
const os = await si.osInfo()
const cpu = await si.cpu()
const cpuTemperature = await si.cpuTemperature()
const currentLoad = await si.currentLoad()
const mem = await si.mem()
const time = si.time()
return {
Platform: `${os.platform} ${os.arch}`,
Distro: `${os.distro} ${os.release}`,
Kernel: os.kernel,
CPU: `${cpu.cores} \u00d7 ${cpu.manufacturer} ${cpu.brand} @ ${cpu.speed.toFixed(2)}GHz`,
'CPU Load': `${currentLoad.currentLoad.toFixed(1)}%`,
'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '),
'CPU Temperature': cpuTemperature && typeof cpuTemperature.main === 'number'
? {
value: cpuTemperature.main,
// Temperature value from this library is hard-coded to Celsius
type: Type.TEMP_CELSIUS
}
: { value: null, type: Type.UNAVAILABLE },
Memory: {
value: {
used: mem.active,
total: mem.total
},
type: Type.BYTE_USAGE
},
Swap: mem && typeof mem.swaptotal === 'number' && mem.swaptotal > 0
? {
value: {
used: mem.swapused,
total: mem.swaptotal
},
type: Type.BYTE_USAGE
}
: { value: null, type: Type.UNAVAILABLE },
Uptime: {
value: Math.floor(time.uptime),
type: Type.UPTIME
}
}
}
self.getServiceInfo = async () => {
const nodeUptime = process.uptime()
/*
if (self.scan.instance) {
try {
self.scan.version = await self.scan.instance.getVersion().then(s => s.trim())
} catch (error) {
logger.error(error)
self.scan.version = 'Errored when querying version.'
}
}
*/
return {
'Node.js': `${process.versions.node}`,
// Scanner: self.scan.version || 'N/A',
'Memory Usage': {
value: process.memoryUsage().rss,
type: Type.BYTE
},
Uptime: {
value: Math.floor(nodeUptime),
type: Type.UPTIME
}
}
}
self.getFileSystems = async () => {
const fsSize = await si.fsSize()
const stats = {}
for (const fs of fsSize) {
const obj = {
value: {
total: fs.size,
used: fs.used
},
type: Type.BYTE_USAGE
}
// "available" is a new attribute in systeminformation v5, only tested on Linux,
// so add an if-check just in case its availability is limited in other platforms
if (typeof fs.available === 'number') {
obj.value.available = fs.available
}
stats[`${fs.fs} (${fs.type}) on ${fs.mount}`] = obj
}
return stats
}
self.getUploadsStats = async db => {
const uploads = await db.table('files')
.select('name', 'type', 'size', 'expirydate')
const stats = {
Total: uploads.length,
Images: 0,
Videos: 0,
Audios: 0,
Others: 0,
Temporary: 0,
'Size in DB': {
value: 0,
type: Type.BYTE
},
'Mime Types': {
value: {},
type: Type.DETAILED
}
}
for (const upload of uploads) {
if (self.imageExtsRegex.test(upload.name)) {
stats.Images++
} else if (self.videoExtsRegex.test(upload.name)) {
stats.Videos++
} else if (self.audioExtsRegex.test(upload.name)) {
stats.Audios++
} else {
stats.Others++
}
if (upload.expirydate !== null) {
stats.Temporary++
}
stats['Size in DB'].value += parseInt(upload.size)
if (stats['Mime Types'].value[upload.type] === undefined) {
stats['Mime Types'].value[upload.type] = 0
}
stats['Mime Types'].value[upload.type]++
}
return stats
}
self.getUsersStats = async db => {
const stats = {
Total: 0,
Disabled: 0,
Usergroups: {
value: {},
type: Type.DETAILED
}
}
const permissionKeys = Object.keys(perms.permissions).reverse()
permissionKeys.forEach(p => {
stats.Usergroups.value[p] = 0
})
const users = await db.table('users')
stats.Total = users.length
for (const user of users) {
if (user.enabled === false || user.enabled === 0) {
stats.Disabled++
}
user.permission = user.permission || 0
for (const p of permissionKeys) {
if (user.permission === perms.permissions[p]) {
stats.Usergroups.value[p]++
break
}
}
}
return stats
}
self.getAlbumsStats = async db => {
const stats = {
Total: 0,
Disabled: 0,
Public: 0,
Downloadable: 0,
'ZIP Generated': 0
}
const albums = await db.table('albums')
stats.Total = albums.length
const activeAlbums = []
for (const album of albums) {
if (!album.enabled) {
stats.Disabled++
continue
}
activeAlbums.push(album.id)
if (album.download) stats.Downloadable++
if (album.public) stats.Public++
}
const files = await jetpack.listAsync(paths.zips)
if (Array.isArray(files)) {
stats['ZIP Generated'] = files.length
}
stats['Files in albums'] = await db.table('files')
.whereIn('albumid', activeAlbums)
.count('id as count')
.then(rows => rows[0].count)
return stats
}
self.statGenerators = {
system: {
title: 'System',
funct: self.getSystemInfo,
maxAge: 1000
},
service: {
title: 'Service',
funct: self.getServiceInfo,
maxAge: 500
},
fileSystems: {
title: 'File Systems',
funct: self.getFileSystems,
maxAge: 60000
},
uploads: {
title: 'Uploads',
funct: self.getUploadsStats,
maxAge: -1
},
users: {
title: 'Users',
funct: self.getUsersStats,
maxAge: -1
},
albums: {
title: 'Albums',
funct: self.getAlbumsStats,
maxAge: -1
}
}
self.statNames = Object.keys(self.statGenerators)
self.generateStats = async db => {
await Promise.all(self.statNames.map(async name => {
const generator = self.statGenerators[name]
if (!self.cachedStats[name]) {
self.cachedStats[name] = {
cache: null,
generating: false,
generatedOn: 0
}
}
// Skip if still generating
if (self.cachedStats[name].generating) return
if (self.cachedStats[name].cache && typeof generator.maxAge === 'number') {
// Skip if maxAge is negative (requires cache to be invaildated via other means),
// or cache still satisfies maxAge
if (generator.maxAge < 0 || (Date.now() - self.cachedStats[name].generatedOn <= generator.maxAge)) {
return
}
}
self.cachedStats[name].generating = true
logger.debug(`${name}: Generating\u2026`)
self.cachedStats[name].cache = await generator.funct(db)
.catch(error => {
logger.error(error)
return null
})
self.cachedStats[name].generatedOn = Date.now()
self.cachedStats[name].generating = false
logger.debug(`${name}: OK`)
}))
return self.statNames.reduce((acc, name) => {
const title = self.statGenerators[name].title
acc[title] = {
...(self.cachedStats[name].cache || {}),
meta: {
cached: Boolean(self.cachedStats[name].cache),
generatedOn: self.cachedStats[name].generatedOn,
maxAge: typeof self.statGenerators[name].maxAge === 'number'
? self.statGenerators[name].maxAge
: null
}
}
return acc
}, {})
}
module.exports = self

View File

@ -8,7 +8,6 @@ const knex = require('knex')
const MarkdownIt = require('markdown-it')
const path = require('path')
const sharp = require('sharp')
const si = require('systeminformation')
const paths = require('./pathsController')
const perms = require('./permissionController')
const Constants = require('./utils/Constants')
@ -145,45 +144,6 @@ if (typeof config.uploads.retentionPeriods === 'object' &&
self.retentions.enabled = true
}
const statsData = {
system: {
title: 'System',
cache: null,
generating: false,
generatedAt: 0
},
service: {
title: 'Service',
cache: null,
generating: false,
generatedAt: 0
},
fileSystems: {
title: 'File Systems',
cache: null,
generating: false,
generatedAt: 0
},
uploads: {
title: 'Uploads',
cache: null,
generating: false,
generatedAt: 0
},
users: {
title: 'Users',
cache: null,
generating: false,
generatedAt: 0
},
albums: {
title: 'Albums',
cache: null,
generating: false,
generatedAt: 0
}
}
// This helper function initiates fetch() with AbortController
// signal controller to handle per-instance global timeout.
// node-fetch's built-in timeout option resets on every redirect,
@ -784,371 +744,21 @@ self.deleteStoredAlbumRenders = albumids => {
}
}
self.invalidateStatsCache = type => {
if (!['albums', 'users', 'uploads'].includes(type)) return
statsData[type].cache = null
}
/** Statistics API **/
const generateStats = async (req, res) => {
self.invalidateStatsCache = StatsManager.invalidateStatsCache
self.stats = async (req, res) => {
const isadmin = perms.is(req.locals.user, 'admin')
if (!isadmin) {
return res.status(403).end()
}
const hrstart = process.hrtime()
const stats = {}
Object.keys(statsData).forEach(key => {
// Pre-assign object keys to fix their display order
stats[statsData[key].title] = {}
})
const os = await si.osInfo()
const getSystemInfo = async () => {
const data = statsData.system
if (!data.cache && data.generating) {
stats[data.title] = false
} else if (((Date.now() - data.generatedAt) <= 1000) || data.generating) {
// Use cache for 1000 ms (1 second)
stats[data.title] = data.cache
} else {
data.generating = true
data.generatedAt = Date.now()
const cpu = await si.cpu()
const cpuTemperature = await si.cpuTemperature()
const currentLoad = await si.currentLoad()
const mem = await si.mem()
const time = si.time()
stats[data.title] = {
Platform: `${os.platform} ${os.arch}`,
Distro: `${os.distro} ${os.release}`,
Kernel: os.kernel,
CPU: `${cpu.cores} \u00d7 ${cpu.manufacturer} ${cpu.brand} @ ${cpu.speed.toFixed(2)}GHz`,
'CPU Load': `${currentLoad.currentLoad.toFixed(1)}%`,
'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '),
'CPU Temperature': {
value: cpuTemperature && typeof cpuTemperature.main === 'number'
? cpuTemperature.main
: 'N/A',
type: 'tempC' // temp value from this library is hard-coded to C
},
Memory: {
value: {
used: mem.active,
total: mem.total
},
type: 'byteUsage'
},
Swap: {
value: typeof mem.swaptotal === 'number' && mem.swaptotal > 0
? {
used: mem.swapused,
total: mem.swaptotal
}
: 'N/A',
type: 'byteUsage'
},
Uptime: {
value: Math.floor(time.uptime),
type: 'uptime'
}
}
// Update cache
data.cache = stats[data.title]
data.generating = false
}
}
const getServiceInfo = async () => {
const data = statsData.service
if (!data.cache && data.generating) {
stats[data.title] = false
} else if (((Date.now() - data.generatedAt) <= 500) || data.generating) {
// Use cache for 500 ms (0.5 seconds)
stats[data.title] = data.cache
} else {
data.generating = true
data.generatedAt = Date.now()
const nodeUptime = process.uptime()
if (self.scan.instance) {
try {
self.scan.version = await self.scan.instance.getVersion().then(s => s.trim())
} catch (error) {
logger.error(error)
self.scan.version = 'Errored when querying version.'
}
}
stats[data.title] = {
'Node.js': `${process.versions.node}`,
Scanner: self.scan.version || 'N/A',
'Memory Usage': {
value: process.memoryUsage().rss,
type: 'byte'
},
Uptime: {
value: Math.floor(nodeUptime),
type: 'uptime'
}
}
// Update cache
data.cache = stats[data.title]
data.generating = false
}
}
const getFileSystems = async () => {
const data = statsData.fileSystems
if (!data.cache && data.generating) {
stats[data.title] = false
} else if (((Date.now() - data.generatedAt) <= 60000) || data.generating) {
// Use cache for 60000 ms (60 seconds)
stats[data.title] = data.cache
} else {
data.generating = true
data.generatedAt = Date.now()
stats[data.title] = {}
const fsSize = await si.fsSize()
for (const fs of fsSize) {
const obj = {
value: {
total: fs.size,
used: fs.used
},
type: 'byteUsage'
}
// "available" is a new attribute in systeminformation v5, only tested on Linux,
// so add an if-check just in case its availability is limited in other platforms
if (typeof fs.available === 'number') {
obj.value.available = fs.available
}
stats[data.title][`${fs.fs} (${fs.type}) on ${fs.mount}`] = obj
}
// Update cache
data.cache = stats[data.title]
data.generating = false
}
}
const getUploadsStats = async () => {
const data = statsData.uploads
if (!data.cache && data.generating) {
stats[data.title] = false
} else if (data.cache) {
// Cache will be invalidated with self.invalidateStatsCache() after any related operations
stats[data.title] = data.cache
} else {
data.generating = true
data.generatedAt = Date.now()
stats[data.title] = {
Total: 0,
Images: 0,
Videos: 0,
Audios: 0,
Others: 0,
Temporary: 0,
'Size in DB': {
value: 0,
type: 'byte'
}
}
const getTotalCountAndSize = async () => {
const uploads = await self.db.table('files')
.select('size')
stats[data.title].Total = uploads.length
stats[data.title]['Size in DB'].value = uploads.reduce((acc, upload) => acc + parseInt(upload.size), 0)
}
const getImagesCount = async () => {
stats[data.title].Images = await self.db.table('files')
.where(function () {
for (const ext of self.imageExts) {
this.orWhere('name', 'like', `%${ext}`)
}
})
.count('id as count')
.then(rows => rows[0].count)
}
const getVideosCount = async () => {
stats[data.title].Videos = await self.db.table('files')
.where(function () {
for (const ext of self.videoExts) {
this.orWhere('name', 'like', `%${ext}`)
}
})
.count('id as count')
.then(rows => rows[0].count)
}
const getAudiosCount = async () => {
stats[data.title].Audios = await self.db.table('files')
.where(function () {
for (const ext of self.audioExts) {
this.orWhere('name', 'like', `%${ext}`)
}
})
.count('id as count')
.then(rows => rows[0].count)
}
const getOthersCount = async () => {
stats[data.title].Temporary = await self.db.table('files')
.whereNotNull('expirydate')
.count('id as count')
.then(rows => rows[0].count)
}
await Promise.all([
getTotalCountAndSize(),
getImagesCount(),
getVideosCount(),
getAudiosCount(),
getOthersCount()
])
stats[data.title].Others = stats[data.title].Total -
stats[data.title].Images -
stats[data.title].Videos -
stats[data.title].Audios
// Update cache
data.cache = stats[data.title]
data.generating = false
}
}
const getUsersStats = async () => {
const data = statsData.users
if (!data.cache && data.generating) {
stats[data.title] = false
} else if (data.cache) {
// Cache will be invalidated with self.invalidateStatsCache() after any related operations
stats[data.title] = data.cache
} else {
data.generating = true
data.generatedAt = Date.now()
stats[data.title] = {
Total: 0,
Disabled: 0
}
const permissionKeys = Object.keys(perms.permissions).reverse()
permissionKeys.forEach(p => {
stats[data.title][p] = 0
})
const users = await self.db.table('users')
stats[data.title].Total = users.length
for (const user of users) {
if (user.enabled === false || user.enabled === 0) {
stats[data.title].Disabled++
}
user.permission = user.permission || 0
for (const p of permissionKeys) {
if (user.permission === perms.permissions[p]) {
stats[data.title][p]++
break
}
}
}
// Update cache
data.cache = stats[data.title]
data.generating = false
}
}
const getAlbumsStats = async () => {
const data = statsData.albums
if (!data.cache && data.generating) {
stats[data.title] = false
} else if (data.cache) {
// Cache will be invalidated with self.invalidateStatsCache() after any related operations
stats[data.title] = data.cache
} else {
data.generating = true
data.generatedAt = Date.now()
stats[data.title] = {
Total: 0,
Disabled: 0,
Public: 0,
Downloadable: 0,
'ZIP Generated': 0
}
const albums = await self.db.table('albums')
stats[data.title].Total = albums.length
const activeAlbums = []
for (const album of albums) {
if (!album.enabled) {
stats[data.title].Disabled++
continue
}
activeAlbums.push(album.id)
if (album.download) stats[data.title].Downloadable++
if (album.public) stats[data.title].Public++
}
const files = await jetpack.listAsync(paths.zips)
if (Array.isArray(files)) {
stats[data.title]['ZIP Generated'] = files.length
}
stats[data.title]['Files in albums'] = await self.db.table('files')
.whereIn('albumid', activeAlbums)
.count('id as count')
.then(rows => rows[0].count)
// Update cache
data.cache = stats[data.title]
data.generating = false
}
}
await Promise.all([
getSystemInfo(),
getServiceInfo(),
getFileSystems(),
getUploadsStats(),
getUsersStats(),
getAlbumsStats()
])
const stats = await StatsManager.generateStats(self.db)
return res.json({ success: true, stats, hrtime: process.hrtime(hrstart) })
}
self.stats = async (req, res) => {
return generateStats(req, res)
.catch(error => {
logger.debug('caught generateStats() errors')
// Reset generating state when encountering any errors
Object.keys(statsData).forEach(key => {
statsData[key].generating = false
})
throw error
})
}
module.exports = self

View File

@ -140,8 +140,8 @@ li[data-action="page-ellipsis"] {
}
}
#statistics tr *:nth-child(1) {
width: 50%
#statistics tr > *:nth-child(1) {
min-width: 33.3%
}
.expirydate {

View File

@ -3103,23 +3103,36 @@ page.getStatistics = (params = {}) => {
let content = ''
const keys = Object.keys(response.data.stats)
for (let i = 0; i < keys.length; i++) {
const meta = []
let rows = ''
if (!response.data.stats[keys[i]]) {
rows += `
<tr>
<td>Generating, please try again later\u2026</td>
<td>Still being generated, please try again later\u2026</td>
<td></td>
</tr>
`
} else {
try {
const valKeys = Object.keys(response.data.stats[keys[i]])
for (let j = 0; j < valKeys.length; j++) {
// Skip meta
if (valKeys[j] === 'meta') {
continue
}
const data = response.data.stats[keys[i]][valKeys[j]]
const type = typeof data === 'object' ? data.type : 'auto'
const value = typeof data === 'object' ? data.value : data
// Skip hidden
if (type === 'hidden') {
continue
}
const value = typeof data === 'object' ? data.value : data
let parsed
switch (type) {
case 'byte':
parsed = page.getPrettyBytes(value)
@ -3136,6 +3149,22 @@ page.getStatistics = (params = {}) => {
}
break
}
case 'detailed':
parsed = `
<table class="table is-narrow is-fullwidth is-hoverable">
<tbody>
${Object.keys(value).map(type => {
return `
<tr>
<td>${type}</td>
<td>${value[type]}</td>
</tr>
`
}).join('\n')}
</tbody>
</table>
`
break
case 'tempC':
// TODO: Unit conversion when required?
parsed = typeof value === 'number'
@ -3165,13 +3194,29 @@ page.getStatistics = (params = {}) => {
</tr>
`
}
const _meta = response.data.stats[keys[i]].meta
if (typeof _meta !== 'undefined') {
// generatedOn
if (typeof _meta.generatedOn !== 'undefined') {
meta.push(`Generated on ${page.getPrettyDate(new Date(_meta.generatedOn))}`)
}
// maxAge
if (typeof _meta.maxAge === 'number') {
if (_meta.maxAge >= 0) {
meta.push(`(${_meta.maxAge / 1000}s)`)
} else {
meta.push('(auto)')
}
}
}
} catch (error) {
rows = `
<tr>
<td>Error parsing response. Try again?</td>
<td></td>
</tr>
`
<tr>
<td>Error parsing response. Try again?</td>
<td></td>
</tr>
`
page.onError(error)
}
}
@ -3182,7 +3227,7 @@ page.getStatistics = (params = {}) => {
<thead>
<tr>
<th class="capitalize">${keys[i]}</th>
<td></td>
<td class="has-text-right">${meta.join(' ')}</td>
</tr>
</thead>
<tbody>