2019-09-08 01:56:29 +00:00
|
|
|
const { promisify } = require('util')
|
2018-09-23 16:28:15 +00:00
|
|
|
const fetch = require('node-fetch')
|
2018-04-13 16:20:57 +00:00
|
|
|
const ffmpeg = require('fluent-ffmpeg')
|
|
|
|
const path = require('path')
|
2018-12-03 07:20:13 +00:00
|
|
|
const sharp = require('sharp')
|
2019-09-08 01:56:29 +00:00
|
|
|
const si = require('systeminformation')
|
Updates (very important to read)
Client-side CSS & JS files will now be processed with Gulp.
Gulp tasks are configured in gulpfile.js file.
CSS files will be optimized with postcss-preset-env, which will
auto-add vendor prefixes and convert any parts necessary for browsers
compatibility.
Afterwards they will be minified with cssnano.
JS files will be optimized with bublé,
likewise for browsers compatibility.
Afterwards they will be minified with terser.
Unprocessed CSS & JS files will now be located at src directory, while
the processed results will be located at dist directory.
Due to bublé, the JS files should now be compatible up to IE 11
at the minimum.
Previously the safe would not work in IE 11 due to extensive usage of
template literals.
Due to that as well, JS files in src directory will now extensively use
arrow functions for my personal comfort (as they will be converted too).
The server will use the processed files at dist directory by default.
If you want to rebuild the files by your own, you can run "yarn build".
Gulp is a development dependency, so make sure you have installed all
development dependencies (e.i. NOT using "yarn install --production").
---
yarn lint -> gulp lint
yarn build -> gulp default
yarn watch -> gulp watch
yarn develop -> env NODE_ENV=development yarn watch
---
Fixed not being able to demote staff into normal users.
/api/token/verify will no longer respond with 401 HTTP error code,
unless an error occurred (which will be 500 HTTP error code).
Fixed /nojs route not displaying file's original name when a duplicate
is found on the server.
Removed is-breeze CSS class name, in favor of Bulma's is-info.
Removed custom styling from auth page, in favor of global styling.
Removed all usage of style HTML attribute in favor of CSS classes.
Renamed js/s/ to js/misc/.
Use loading spinners on dashboard's sidebar menus.
Disable all other sidebar menus when something is loading.
Changed title HTML attribute of disabled control buttons in
uploads & users list.
Hid checkboxes and WIP controls from users list.
Better error messages handling.
Especially homepage will now support CF's HTTP error codes.
Updated various icons.
Also, added fontello config file at public/libs/fontello/config.json.
This should let you edit them more easily with fontello.
Use Gatsby icon for my blog's link in homepage's footer.
A bunch of other improvements here & there.
2019-09-15 06:20:11 +00:00
|
|
|
const paths = require('./pathsController')
|
|
|
|
const perms = require('./permissionController')
|
2021-01-08 03:48:08 +00:00
|
|
|
const apiErrorsHandler = require('./handlers/apiErrorsHandler')
|
|
|
|
const ClientError = require('./utils/ClientError')
|
Updates (very important to read)
Client-side CSS & JS files will now be processed with Gulp.
Gulp tasks are configured in gulpfile.js file.
CSS files will be optimized with postcss-preset-env, which will
auto-add vendor prefixes and convert any parts necessary for browsers
compatibility.
Afterwards they will be minified with cssnano.
JS files will be optimized with bublé,
likewise for browsers compatibility.
Afterwards they will be minified with terser.
Unprocessed CSS & JS files will now be located at src directory, while
the processed results will be located at dist directory.
Due to bublé, the JS files should now be compatible up to IE 11
at the minimum.
Previously the safe would not work in IE 11 due to extensive usage of
template literals.
Due to that as well, JS files in src directory will now extensively use
arrow functions for my personal comfort (as they will be converted too).
The server will use the processed files at dist directory by default.
If you want to rebuild the files by your own, you can run "yarn build".
Gulp is a development dependency, so make sure you have installed all
development dependencies (e.i. NOT using "yarn install --production").
---
yarn lint -> gulp lint
yarn build -> gulp default
yarn watch -> gulp watch
yarn develop -> env NODE_ENV=development yarn watch
---
Fixed not being able to demote staff into normal users.
/api/token/verify will no longer respond with 401 HTTP error code,
unless an error occurred (which will be 500 HTTP error code).
Fixed /nojs route not displaying file's original name when a duplicate
is found on the server.
Removed is-breeze CSS class name, in favor of Bulma's is-info.
Removed custom styling from auth page, in favor of global styling.
Removed all usage of style HTML attribute in favor of CSS classes.
Renamed js/s/ to js/misc/.
Use loading spinners on dashboard's sidebar menus.
Disable all other sidebar menus when something is loading.
Changed title HTML attribute of disabled control buttons in
uploads & users list.
Hid checkboxes and WIP controls from users list.
Better error messages handling.
Especially homepage will now support CF's HTTP error codes.
Updated various icons.
Also, added fontello config file at public/libs/fontello/config.json.
This should let you edit them more easily with fontello.
Use Gatsby icon for my blog's link in homepage's footer.
A bunch of other improvements here & there.
2019-09-15 06:20:11 +00:00
|
|
|
const config = require('./../config')
|
|
|
|
const logger = require('./../logger')
|
|
|
|
const db = require('knex')(config.database)
|
2017-03-17 04:14:10 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
const self = {
|
2020-10-31 23:35:56 +00:00
|
|
|
clamscan: {
|
|
|
|
instance: null,
|
|
|
|
version: null,
|
2020-04-04 14:20:01 +00:00
|
|
|
groupBypass: config.uploads.scan.groupBypass || null,
|
|
|
|
whitelistExtensions: (Array.isArray(config.uploads.scan.whitelistExtensions) &&
|
2020-10-30 18:12:09 +00:00
|
|
|
config.uploads.scan.whitelistExtensions.length)
|
|
|
|
? config.uploads.scan.whitelistExtensions
|
|
|
|
: null,
|
2020-04-04 14:20:01 +00:00
|
|
|
maxSize: (parseInt(config.uploads.scan.maxSize) * 1e6) || null
|
2019-09-08 01:56:29 +00:00
|
|
|
},
|
|
|
|
gitHash: null,
|
|
|
|
idSet: null,
|
|
|
|
|
|
|
|
idMaxTries: config.uploads.maxTries || 1,
|
|
|
|
|
2020-05-26 03:47:24 +00:00
|
|
|
imageExts: ['.gif', '.jpeg', '.jpg', '.png', '.svg', '.tif', '.tiff', '.webp'],
|
|
|
|
videoExts: ['.3g2', '.3gp', '.asf', '.avchd', '.avi', '.divx', '.evo', '.flv', '.h264', '.h265', '.hevc', '.m2p', '.m2ts', '.m4v', '.mk3d', '.mkv', '.mov', '.mp4', '.mpeg', '.mpg', '.mxf', '.ogg', '.ogv', '.ps', '.qt', '.rmvb', '.ts', '.vob', '.webm', '.wmv'],
|
2020-11-03 15:51:29 +00:00
|
|
|
audioExts: ['.flac', '.mp3', '.wav', '.wma'],
|
2019-09-08 01:56:29 +00:00
|
|
|
|
2020-06-06 12:43:20 +00:00
|
|
|
thumbsSize: config.uploads.generateThumbs.size || 200,
|
2019-09-17 04:13:41 +00:00
|
|
|
ffprobe: promisify(ffmpeg.ffprobe),
|
|
|
|
|
2020-04-18 19:52:11 +00:00
|
|
|
albumsCache: {},
|
|
|
|
timezoneOffset: new Date().getTimezoneOffset()
|
2019-09-08 01:56:29 +00:00
|
|
|
}
|
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
const statsData = {
|
2019-04-12 00:45:33 +00:00
|
|
|
system: {
|
2020-12-25 14:06:21 +00:00
|
|
|
title: 'System',
|
2019-04-12 00:45:33 +00:00
|
|
|
cache: null,
|
2019-11-14 08:08:56 +00:00
|
|
|
generating: false,
|
|
|
|
generatedAt: 0
|
2019-04-12 00:45:33 +00:00
|
|
|
},
|
2020-12-25 14:06:21 +00:00
|
|
|
fileSystems: {
|
|
|
|
title: 'File Systems',
|
2020-05-02 12:28:13 +00:00
|
|
|
cache: null,
|
|
|
|
generating: false,
|
|
|
|
generatedAt: 0
|
|
|
|
},
|
2020-12-25 14:06:21 +00:00
|
|
|
uploads: {
|
|
|
|
title: 'Uploads',
|
2019-04-12 00:45:33 +00:00
|
|
|
cache: null,
|
2019-06-03 19:40:24 +00:00
|
|
|
generating: false,
|
Improved albums public page cache and more
Removed its dependency towards albums' editedAt property.
Editing album's metas (name, description, etc) will no longer update
its editedAt property.
Instead it will now ONLY be updated when adding/removing files to/from
it. Just like how it was meant to be, which was to be used to check
whether it's necessary to re-generate their downloadable ZIPs.
Albums public page cache will still be properly invalidated when
adding/removing files to/from it, as well as after editing their metas.
Added views/album-notice.njk to be used to render okay-ish notice when
an album's public page is still being generated.
I was originally thinking of using it for disabled albums as well, but
I refrained from it to reduce the possibility of disabled album IDs from
being easily scanned (as it just returns 404 now).
Removed invalidatedAt property from stats cache. Instead their caches
will immediately be nullified as they should (thus frees up memory
slightly as well).
Stats cache for albums will now only be cleared when truly necessary.
As in, adding/removing files to/from albums will no longer clear them.
Updated Nunjucks files to properly use h1, h2, h3 tags in actual
hierarchical orders.
Elements that don't need to use hX tags will now use P instead.
Nothing changes visually, only structurally.
Fixed some elements in Nunjucks using single quotes instead of
double quotes. They'd have worked the same, but consistency.
Added h1 title in FAQ page.
Make text for no JS warning a bit bigger, and improved the phrasing
a little bit.
2020-06-03 03:44:24 +00:00
|
|
|
generatedAt: 0
|
2019-04-12 00:45:33 +00:00
|
|
|
},
|
|
|
|
users: {
|
2020-12-25 14:06:21 +00:00
|
|
|
title: 'Users',
|
2019-04-12 00:45:33 +00:00
|
|
|
cache: null,
|
2019-06-03 19:40:24 +00:00
|
|
|
generating: false,
|
Improved albums public page cache and more
Removed its dependency towards albums' editedAt property.
Editing album's metas (name, description, etc) will no longer update
its editedAt property.
Instead it will now ONLY be updated when adding/removing files to/from
it. Just like how it was meant to be, which was to be used to check
whether it's necessary to re-generate their downloadable ZIPs.
Albums public page cache will still be properly invalidated when
adding/removing files to/from it, as well as after editing their metas.
Added views/album-notice.njk to be used to render okay-ish notice when
an album's public page is still being generated.
I was originally thinking of using it for disabled albums as well, but
I refrained from it to reduce the possibility of disabled album IDs from
being easily scanned (as it just returns 404 now).
Removed invalidatedAt property from stats cache. Instead their caches
will immediately be nullified as they should (thus frees up memory
slightly as well).
Stats cache for albums will now only be cleared when truly necessary.
As in, adding/removing files to/from albums will no longer clear them.
Updated Nunjucks files to properly use h1, h2, h3 tags in actual
hierarchical orders.
Elements that don't need to use hX tags will now use P instead.
Nothing changes visually, only structurally.
Fixed some elements in Nunjucks using single quotes instead of
double quotes. They'd have worked the same, but consistency.
Added h1 title in FAQ page.
Make text for no JS warning a bit bigger, and improved the phrasing
a little bit.
2020-06-03 03:44:24 +00:00
|
|
|
generatedAt: 0
|
2019-04-12 00:45:33 +00:00
|
|
|
},
|
2020-12-25 14:06:21 +00:00
|
|
|
albums: {
|
|
|
|
title: 'Albums',
|
2019-04-12 00:45:33 +00:00
|
|
|
cache: null,
|
2019-06-03 19:40:24 +00:00
|
|
|
generating: false,
|
Improved albums public page cache and more
Removed its dependency towards albums' editedAt property.
Editing album's metas (name, description, etc) will no longer update
its editedAt property.
Instead it will now ONLY be updated when adding/removing files to/from
it. Just like how it was meant to be, which was to be used to check
whether it's necessary to re-generate their downloadable ZIPs.
Albums public page cache will still be properly invalidated when
adding/removing files to/from it, as well as after editing their metas.
Added views/album-notice.njk to be used to render okay-ish notice when
an album's public page is still being generated.
I was originally thinking of using it for disabled albums as well, but
I refrained from it to reduce the possibility of disabled album IDs from
being easily scanned (as it just returns 404 now).
Removed invalidatedAt property from stats cache. Instead their caches
will immediately be nullified as they should (thus frees up memory
slightly as well).
Stats cache for albums will now only be cleared when truly necessary.
As in, adding/removing files to/from albums will no longer clear them.
Updated Nunjucks files to properly use h1, h2, h3 tags in actual
hierarchical orders.
Elements that don't need to use hX tags will now use P instead.
Nothing changes visually, only structurally.
Fixed some elements in Nunjucks using single quotes instead of
double quotes. They'd have worked the same, but consistency.
Added h1 title in FAQ page.
Make text for no JS warning a bit bigger, and improved the phrasing
a little bit.
2020-06-03 03:44:24 +00:00
|
|
|
generatedAt: 0
|
2019-04-12 00:45:33 +00:00
|
|
|
}
|
|
|
|
}
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2020-06-26 06:48:15 +00:00
|
|
|
const cloudflareAuth = config.cloudflare && config.cloudflare.zoneId &&
|
|
|
|
(config.cloudflare.apiToken || config.cloudflare.userServiceKey ||
|
|
|
|
(config.cloudflare.apiKey && config.cloudflare.email))
|
2018-04-29 12:47:24 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.mayGenerateThumb = extname => {
|
|
|
|
return (config.uploads.generateThumbs.image && self.imageExts.includes(extname)) ||
|
|
|
|
(config.uploads.generateThumbs.video && self.videoExts.includes(extname))
|
2018-04-29 12:47:24 +00:00
|
|
|
}
|
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
// Expand if necessary (must be lower case); for now only preserves some known tarballs
|
|
|
|
const extPreserves = ['.tar.gz', '.tar.z', '.tar.bz2', '.tar.lzma', '.tar.lzo', '.tar.xz']
|
2018-09-17 19:32:27 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.extname = filename => {
|
2018-11-28 17:52:12 +00:00
|
|
|
// Always return blank string if the filename does not seem to have a valid extension
|
|
|
|
// Files such as .DS_Store (anything that starts with a dot, without any extension after) will still be accepted
|
2018-12-18 17:01:28 +00:00
|
|
|
if (!/\../.test(filename)) return ''
|
2018-11-28 17:52:12 +00:00
|
|
|
|
2018-09-17 19:32:27 +00:00
|
|
|
let lower = filename.toLowerCase() // due to this, the returned extname will always be lower case
|
|
|
|
let multi = ''
|
|
|
|
let extname = ''
|
|
|
|
|
|
|
|
// check for multi-archive extensions (.001, .002, and so on)
|
|
|
|
if (/\.\d{3}$/.test(lower)) {
|
|
|
|
multi = lower.slice(lower.lastIndexOf('.') - lower.length)
|
|
|
|
lower = lower.slice(0, lower.lastIndexOf('.'))
|
|
|
|
}
|
|
|
|
|
|
|
|
// check against extensions that must be preserved
|
2020-10-30 18:12:09 +00:00
|
|
|
for (const extPreserve of extPreserves) {
|
2019-09-08 01:56:29 +00:00
|
|
|
if (lower.endsWith(extPreserve)) {
|
|
|
|
extname = extPreserve
|
2018-09-17 19:32:27 +00:00
|
|
|
break
|
|
|
|
}
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2018-09-17 19:32:27 +00:00
|
|
|
|
2020-10-30 18:12:09 +00:00
|
|
|
if (!extname) {
|
2018-09-17 19:32:27 +00:00
|
|
|
extname = lower.slice(lower.lastIndexOf('.') - lower.length) // path.extname(lower)
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2018-09-17 19:32:27 +00:00
|
|
|
|
|
|
|
return extname + multi
|
|
|
|
}
|
|
|
|
|
2020-04-18 19:52:11 +00:00
|
|
|
self.escape = string => {
|
2018-12-13 09:09:46 +00:00
|
|
|
// MIT License
|
|
|
|
// Copyright(c) 2012-2013 TJ Holowaychuk
|
|
|
|
// Copyright(c) 2015 Andreas Lubbe
|
|
|
|
// Copyright(c) 2015 Tiancheng "Timothy" Gu
|
|
|
|
|
2020-10-30 18:12:09 +00:00
|
|
|
if (!string) return string
|
2018-12-13 09:09:46 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
const str = String(string)
|
2018-12-13 09:09:46 +00:00
|
|
|
const match = /["'&<>]/.exec(str)
|
|
|
|
|
2020-10-30 18:12:09 +00:00
|
|
|
if (!match) return str
|
2018-12-13 09:09:46 +00:00
|
|
|
|
|
|
|
let escape
|
|
|
|
let html = ''
|
|
|
|
let index = 0
|
|
|
|
let lastIndex = 0
|
|
|
|
|
|
|
|
for (index = match.index; index < str.length; index++) {
|
|
|
|
switch (str.charCodeAt(index)) {
|
|
|
|
case 34: // "
|
|
|
|
escape = '"'
|
|
|
|
break
|
|
|
|
case 38: // &
|
|
|
|
escape = '&'
|
|
|
|
break
|
|
|
|
case 39: // '
|
|
|
|
escape = '''
|
|
|
|
break
|
|
|
|
case 60: // <
|
|
|
|
escape = '<'
|
|
|
|
break
|
|
|
|
case 62: // >
|
|
|
|
escape = '>'
|
|
|
|
break
|
|
|
|
default:
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-10-30 18:12:09 +00:00
|
|
|
if (lastIndex !== index) {
|
2018-12-13 09:09:46 +00:00
|
|
|
html += str.substring(lastIndex, index)
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2018-12-13 09:09:46 +00:00
|
|
|
|
|
|
|
lastIndex = index + 1
|
|
|
|
html += escape
|
|
|
|
}
|
|
|
|
|
|
|
|
return lastIndex !== index
|
|
|
|
? html + str.substring(lastIndex, index)
|
|
|
|
: html
|
|
|
|
}
|
|
|
|
|
Updates (very important to read)
Client-side CSS & JS files will now be processed with Gulp.
Gulp tasks are configured in gulpfile.js file.
CSS files will be optimized with postcss-preset-env, which will
auto-add vendor prefixes and convert any parts necessary for browsers
compatibility.
Afterwards they will be minified with cssnano.
JS files will be optimized with bublé,
likewise for browsers compatibility.
Afterwards they will be minified with terser.
Unprocessed CSS & JS files will now be located at src directory, while
the processed results will be located at dist directory.
Due to bublé, the JS files should now be compatible up to IE 11
at the minimum.
Previously the safe would not work in IE 11 due to extensive usage of
template literals.
Due to that as well, JS files in src directory will now extensively use
arrow functions for my personal comfort (as they will be converted too).
The server will use the processed files at dist directory by default.
If you want to rebuild the files by your own, you can run "yarn build".
Gulp is a development dependency, so make sure you have installed all
development dependencies (e.i. NOT using "yarn install --production").
---
yarn lint -> gulp lint
yarn build -> gulp default
yarn watch -> gulp watch
yarn develop -> env NODE_ENV=development yarn watch
---
Fixed not being able to demote staff into normal users.
/api/token/verify will no longer respond with 401 HTTP error code,
unless an error occurred (which will be 500 HTTP error code).
Fixed /nojs route not displaying file's original name when a duplicate
is found on the server.
Removed is-breeze CSS class name, in favor of Bulma's is-info.
Removed custom styling from auth page, in favor of global styling.
Removed all usage of style HTML attribute in favor of CSS classes.
Renamed js/s/ to js/misc/.
Use loading spinners on dashboard's sidebar menus.
Disable all other sidebar menus when something is loading.
Changed title HTML attribute of disabled control buttons in
uploads & users list.
Hid checkboxes and WIP controls from users list.
Better error messages handling.
Especially homepage will now support CF's HTTP error codes.
Updated various icons.
Also, added fontello config file at public/libs/fontello/config.json.
This should let you edit them more easily with fontello.
Use Gatsby icon for my blog's link in homepage's footer.
A bunch of other improvements here & there.
2019-09-15 06:20:11 +00:00
|
|
|
self.stripIndents = string => {
|
|
|
|
if (!string) return
|
|
|
|
const result = string.replace(/^[^\S\n]+/gm, '')
|
|
|
|
const match = result.match(/^[^\S\n]*(?=\S)/gm)
|
|
|
|
const indent = match && Math.min(...match.map(el => el.length))
|
|
|
|
if (indent) {
|
|
|
|
const regexp = new RegExp(`^.{${indent}}`, 'gm')
|
|
|
|
return result.replace(regexp, '')
|
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
2021-01-08 03:48:08 +00:00
|
|
|
self.assertUser = async token => {
|
|
|
|
const user = await db.table('users')
|
|
|
|
.where('token', token)
|
|
|
|
.first()
|
|
|
|
if (user) {
|
|
|
|
if (user.enabled === false || user.enabled === 0) {
|
|
|
|
throw new ClientError('This account has been disabled.', { statusCode: 403 })
|
2018-10-09 19:52:41 +00:00
|
|
|
}
|
2021-01-08 03:48:08 +00:00
|
|
|
return user
|
|
|
|
} else {
|
|
|
|
throw new ClientError('Invalid token.', { statusCode: 403 })
|
|
|
|
}
|
|
|
|
}
|
2018-04-29 12:47:24 +00:00
|
|
|
|
2021-01-08 03:48:08 +00:00
|
|
|
self.authorize = async req => {
|
|
|
|
const token = req.headers.token
|
|
|
|
if (token === undefined) {
|
|
|
|
throw new ClientError('No token provided.', { statusCode: 403 })
|
2019-09-08 01:56:29 +00:00
|
|
|
}
|
2021-01-08 03:48:08 +00:00
|
|
|
return self.assertUser(token)
|
2018-01-23 20:06:30 +00:00
|
|
|
}
|
2017-10-04 00:13:38 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.generateThumbs = async (name, extname, force) => {
|
|
|
|
const thumbname = path.join(paths.thumbs, name.slice(0, -extname.length) + '.png')
|
|
|
|
|
|
|
|
try {
|
|
|
|
// Check if thumbnail already exists
|
|
|
|
try {
|
|
|
|
const lstat = await paths.lstat(thumbname)
|
2020-10-30 18:12:09 +00:00
|
|
|
if (lstat.isSymbolicLink()) {
|
2019-09-08 01:56:29 +00:00
|
|
|
// Unlink if symlink (should be symlink to the placeholder)
|
|
|
|
await paths.unlink(thumbname)
|
2020-10-30 18:12:09 +00:00
|
|
|
} else if (!force) {
|
2019-09-08 01:56:29 +00:00
|
|
|
// Continue only if it does not exist, unless forced to
|
|
|
|
return true
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
} catch (error) {
|
2021-01-08 02:56:01 +00:00
|
|
|
// Re-throw non-ENOENT error
|
2020-10-30 18:12:09 +00:00
|
|
|
if (error.code !== 'ENOENT') throw error
|
2019-09-08 01:56:29 +00:00
|
|
|
}
|
2018-05-12 14:01:14 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
// Full path to input file
|
|
|
|
const input = path.join(paths.uploads, name)
|
|
|
|
|
|
|
|
// If image extension
|
|
|
|
if (self.imageExts.includes(extname)) {
|
|
|
|
const resizeOptions = {
|
2020-06-06 12:43:20 +00:00
|
|
|
width: self.thumbsSize,
|
|
|
|
height: self.thumbsSize,
|
2019-09-08 01:56:29 +00:00
|
|
|
fit: 'contain',
|
|
|
|
background: {
|
|
|
|
r: 0,
|
|
|
|
g: 0,
|
|
|
|
b: 0,
|
|
|
|
alpha: 0
|
|
|
|
}
|
|
|
|
}
|
|
|
|
const image = sharp(input)
|
|
|
|
const metadata = await image.metadata()
|
|
|
|
if (metadata.width > resizeOptions.width || metadata.height > resizeOptions.height) {
|
|
|
|
await image
|
|
|
|
.resize(resizeOptions)
|
|
|
|
.toFile(thumbname)
|
|
|
|
} else if (metadata.width === resizeOptions.width && metadata.height === resizeOptions.height) {
|
|
|
|
await image
|
|
|
|
.toFile(thumbname)
|
|
|
|
} else {
|
|
|
|
const x = resizeOptions.width - metadata.width
|
|
|
|
const y = resizeOptions.height - metadata.height
|
|
|
|
await image
|
|
|
|
.extend({
|
|
|
|
top: Math.floor(y / 2),
|
|
|
|
bottom: Math.ceil(y / 2),
|
|
|
|
left: Math.floor(x / 2),
|
|
|
|
right: Math.ceil(x / 2),
|
|
|
|
background: resizeOptions.background
|
2018-09-04 17:29:53 +00:00
|
|
|
})
|
2019-09-08 01:56:29 +00:00
|
|
|
.toFile(thumbname)
|
2018-09-04 17:29:53 +00:00
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
} else if (self.videoExts.includes(extname)) {
|
|
|
|
const metadata = await self.ffprobe(input)
|
2020-06-15 14:04:30 +00:00
|
|
|
|
2019-10-15 10:53:23 +00:00
|
|
|
const duration = parseInt(metadata.format.duration)
|
2020-10-30 18:12:09 +00:00
|
|
|
if (isNaN(duration)) {
|
2021-01-08 02:56:01 +00:00
|
|
|
throw new Error('File does not have valid duration metadata')
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2018-09-04 17:29:53 +00:00
|
|
|
|
2020-06-15 14:04:30 +00:00
|
|
|
const videoStream = metadata.streams && metadata.streams.find(s => s.codec_type === 'video')
|
2020-10-30 18:12:09 +00:00
|
|
|
if (!videoStream || !videoStream.width || !videoStream.height) {
|
2021-01-08 02:56:01 +00:00
|
|
|
throw new Error('File does not have valid video stream metadata')
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-08-23 13:31:44 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
await new Promise((resolve, reject) => {
|
|
|
|
ffmpeg(input)
|
2020-06-15 14:04:30 +00:00
|
|
|
.on('error', error => reject(error))
|
|
|
|
.on('end', () => resolve())
|
|
|
|
.screenshots({
|
|
|
|
folder: paths.thumbs,
|
|
|
|
filename: name.slice(0, -extname.length) + '.png',
|
|
|
|
timestamps: ['20%'],
|
|
|
|
size: videoStream.width >= videoStream.height
|
|
|
|
? `${self.thumbsSize}x?`
|
|
|
|
: `?x${self.thumbsSize}`
|
2018-05-12 14:01:14 +00:00
|
|
|
})
|
2019-09-08 01:56:29 +00:00
|
|
|
})
|
2020-06-15 14:04:30 +00:00
|
|
|
.catch(error => error) // Error passthrough
|
|
|
|
.then(async error => {
|
|
|
|
// FFMPEG would just warn instead of exiting with errors when dealing with incomplete files
|
|
|
|
// Sometimes FFMPEG would throw errors but actually somehow succeeded in making the thumbnails
|
|
|
|
// (this could be a fallback mechanism of fluent-ffmpeg library instead)
|
|
|
|
// So instead we check if the thumbnail exists to really make sure
|
|
|
|
try {
|
|
|
|
await paths.lstat(thumbname)
|
|
|
|
return true
|
|
|
|
} catch (err) {
|
2020-10-30 18:12:09 +00:00
|
|
|
if (err.code === 'ENOENT') {
|
2021-01-08 02:56:01 +00:00
|
|
|
throw error || new Error('FFMPEG exited with empty output file')
|
2020-10-30 18:12:09 +00:00
|
|
|
} else {
|
2020-06-15 14:04:30 +00:00
|
|
|
throw error || err
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2020-06-15 14:04:30 +00:00
|
|
|
}
|
|
|
|
})
|
2019-09-08 01:56:29 +00:00
|
|
|
} else {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
} catch (error) {
|
2020-06-15 14:04:30 +00:00
|
|
|
logger.error(`[${name}]: ${error.toString().trim()}`)
|
2019-09-08 01:56:29 +00:00
|
|
|
try {
|
2021-01-08 03:48:08 +00:00
|
|
|
await paths.unlink(thumbname).catch(() => {}) // try to unlink incomplete thumbs first
|
2019-09-08 01:56:29 +00:00
|
|
|
await paths.symlink(paths.thumbPlaceholder, thumbname)
|
|
|
|
return true
|
|
|
|
} catch (err) {
|
|
|
|
logger.error(err)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
2018-01-23 20:06:30 +00:00
|
|
|
}
|
2017-03-17 04:14:10 +00:00
|
|
|
|
2019-11-29 13:42:53 +00:00
|
|
|
self.stripTags = async (name, extname) => {
|
|
|
|
const fullpath = path.join(paths.uploads, name)
|
2021-01-08 03:48:08 +00:00
|
|
|
let tmpfile
|
2019-11-29 13:42:53 +00:00
|
|
|
|
2021-01-08 03:48:08 +00:00
|
|
|
try {
|
|
|
|
if (self.imageExts.includes(extname)) {
|
|
|
|
tmpfile = path.join(paths.uploads, `tmp-${name}`)
|
|
|
|
await paths.rename(fullpath, tmpfile)
|
2019-11-29 13:42:53 +00:00
|
|
|
await sharp(tmpfile)
|
|
|
|
.toFile(fullpath)
|
2021-01-08 03:48:08 +00:00
|
|
|
} else if (config.uploads.stripTags.video && self.videoExts.includes(extname)) {
|
|
|
|
tmpfile = path.join(paths.uploads, `tmp-${name}`)
|
|
|
|
await paths.rename(fullpath, tmpfile)
|
2019-11-29 13:42:53 +00:00
|
|
|
await new Promise((resolve, reject) => {
|
|
|
|
ffmpeg(tmpfile)
|
|
|
|
.output(fullpath)
|
|
|
|
.outputOptions([
|
|
|
|
// Experimental.
|
|
|
|
'-c copy',
|
|
|
|
'-map_metadata:g -1:g',
|
|
|
|
'-map_metadata:s:v -1:g',
|
|
|
|
'-map_metadata:s:a -1:g'
|
|
|
|
])
|
|
|
|
.on('error', error => reject(error))
|
|
|
|
.on('end', () => resolve(true))
|
|
|
|
.run()
|
|
|
|
})
|
2021-01-08 03:48:08 +00:00
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
logger.error(`[${name}]: ${error.toString().trim()}`)
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tmpfile) {
|
|
|
|
try {
|
2019-11-29 13:42:53 +00:00
|
|
|
await paths.unlink(tmpfile)
|
|
|
|
} catch (error) {
|
2021-01-08 03:48:08 +00:00
|
|
|
if (error.code !== 'ENOENT') {
|
|
|
|
logger.error(`[${name}]: ${error.toString().trim()}`)
|
|
|
|
}
|
2019-11-29 13:42:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.unlinkFile = async (filename, predb) => {
|
|
|
|
try {
|
|
|
|
await paths.unlink(path.join(paths.uploads, filename))
|
|
|
|
} catch (error) {
|
2021-01-08 03:48:08 +00:00
|
|
|
// Re-throw non-ENOENT error
|
2020-10-30 18:12:09 +00:00
|
|
|
if (error.code !== 'ENOENT') throw error
|
2019-09-08 01:56:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const identifier = filename.split('.')[0]
|
|
|
|
|
|
|
|
// Do not remove from identifiers cache on pre-db-deletion
|
|
|
|
if (!predb && self.idSet) {
|
|
|
|
self.idSet.delete(identifier)
|
|
|
|
// logger.log(`Removed ${identifier} from identifiers cache (deleteFile)`)
|
|
|
|
}
|
|
|
|
|
|
|
|
const extname = self.extname(filename)
|
2020-10-30 18:12:09 +00:00
|
|
|
if (self.imageExts.includes(extname) || self.videoExts.includes(extname)) {
|
2019-09-08 01:56:29 +00:00
|
|
|
try {
|
|
|
|
await paths.unlink(path.join(paths.thumbs, `${identifier}.png`))
|
|
|
|
} catch (error) {
|
2021-01-08 03:48:08 +00:00
|
|
|
// Re-throw non-ENOENT error
|
2020-10-30 18:12:09 +00:00
|
|
|
if (error.code !== 'ENOENT') throw error
|
2019-09-08 01:56:29 +00:00
|
|
|
}
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2018-03-30 02:39:53 +00:00
|
|
|
}
|
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.bulkDeleteFromDb = async (field, values, user) => {
|
2019-09-23 08:09:15 +00:00
|
|
|
// Always return an empty array on failure
|
2020-10-30 18:12:09 +00:00
|
|
|
if (!user || !['id', 'name'].includes(field) || !values.length) return []
|
2018-05-10 17:25:52 +00:00
|
|
|
|
2019-02-05 03:36:14 +00:00
|
|
|
// SQLITE_LIMIT_VARIABLE_NUMBER, which defaults to 999
|
|
|
|
// Read more: https://www.sqlite.org/limits.html
|
|
|
|
const MAX_VARIABLES_CHUNK_SIZE = 999
|
|
|
|
const chunks = []
|
2020-10-30 18:12:09 +00:00
|
|
|
while (values.length) {
|
2019-09-08 01:56:29 +00:00
|
|
|
chunks.push(values.splice(0, MAX_VARIABLES_CHUNK_SIZE))
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-02-05 03:36:14 +00:00
|
|
|
|
2020-10-11 10:32:22 +00:00
|
|
|
const failed = []
|
2018-10-13 11:06:58 +00:00
|
|
|
const ismoderator = perms.is(user, 'moderator')
|
2018-05-10 17:25:52 +00:00
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
try {
|
2020-10-11 10:32:22 +00:00
|
|
|
const unlinkeds = []
|
2019-09-08 01:56:29 +00:00
|
|
|
const albumids = []
|
2018-05-09 09:53:27 +00:00
|
|
|
|
2019-09-23 08:09:15 +00:00
|
|
|
await Promise.all(chunks.map(async chunk => {
|
2019-09-08 01:56:29 +00:00
|
|
|
const files = await db.table('files')
|
2019-09-23 08:09:15 +00:00
|
|
|
.whereIn(field, chunk)
|
2019-09-08 01:56:29 +00:00
|
|
|
.where(function () {
|
2020-10-30 18:12:09 +00:00
|
|
|
if (!ismoderator) {
|
2019-09-08 18:33:07 +00:00
|
|
|
this.where('userid', user.id)
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
// Push files that could not be found in db
|
2020-10-11 10:32:22 +00:00
|
|
|
failed.push(...chunk.filter(value => !files.find(file => file[field] === value)))
|
2019-09-08 01:56:29 +00:00
|
|
|
|
|
|
|
// Unlink all found files
|
|
|
|
const unlinked = []
|
2019-09-23 08:09:15 +00:00
|
|
|
|
|
|
|
await Promise.all(files.map(async file => {
|
2019-09-08 01:56:29 +00:00
|
|
|
try {
|
|
|
|
await self.unlinkFile(file.name, true)
|
|
|
|
unlinked.push(file)
|
|
|
|
} catch (error) {
|
|
|
|
logger.error(error)
|
|
|
|
failed.push(file[field])
|
2019-08-20 02:16:34 +00:00
|
|
|
}
|
2019-09-23 08:09:15 +00:00
|
|
|
}))
|
2019-08-20 02:16:34 +00:00
|
|
|
|
2019-09-23 08:09:15 +00:00
|
|
|
if (!unlinked.length) return
|
2019-09-08 01:56:29 +00:00
|
|
|
|
|
|
|
// Delete all unlinked files from db
|
|
|
|
await db.table('files')
|
|
|
|
.whereIn('id', unlinked.map(file => file.id))
|
|
|
|
.del()
|
|
|
|
self.invalidateStatsCache('uploads')
|
|
|
|
|
2020-10-30 18:12:09 +00:00
|
|
|
if (self.idSet) {
|
2019-09-08 01:56:29 +00:00
|
|
|
unlinked.forEach(file => {
|
|
|
|
const identifier = file.name.split('.')[0]
|
|
|
|
self.idSet.delete(identifier)
|
|
|
|
// logger.log(`Removed ${identifier} from identifiers cache (bulkDeleteFromDb)`)
|
|
|
|
})
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
|
|
|
|
// Push album ids
|
|
|
|
unlinked.forEach(file => {
|
2020-10-30 18:12:09 +00:00
|
|
|
if (file.albumid && !albumids.includes(file.albumid)) {
|
2019-09-08 01:56:29 +00:00
|
|
|
albumids.push(file.albumid)
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
// Push unlinked files
|
2020-10-11 10:32:22 +00:00
|
|
|
unlinkeds.push(...unlinked)
|
2019-09-23 08:09:15 +00:00
|
|
|
}))
|
2019-09-08 01:56:29 +00:00
|
|
|
|
|
|
|
if (unlinkeds.length) {
|
|
|
|
// Update albums if necessary, but do not wait
|
Improved albums public page cache and more
Removed its dependency towards albums' editedAt property.
Editing album's metas (name, description, etc) will no longer update
its editedAt property.
Instead it will now ONLY be updated when adding/removing files to/from
it. Just like how it was meant to be, which was to be used to check
whether it's necessary to re-generate their downloadable ZIPs.
Albums public page cache will still be properly invalidated when
adding/removing files to/from it, as well as after editing their metas.
Added views/album-notice.njk to be used to render okay-ish notice when
an album's public page is still being generated.
I was originally thinking of using it for disabled albums as well, but
I refrained from it to reduce the possibility of disabled album IDs from
being easily scanned (as it just returns 404 now).
Removed invalidatedAt property from stats cache. Instead their caches
will immediately be nullified as they should (thus frees up memory
slightly as well).
Stats cache for albums will now only be cleared when truly necessary.
As in, adding/removing files to/from albums will no longer clear them.
Updated Nunjucks files to properly use h1, h2, h3 tags in actual
hierarchical orders.
Elements that don't need to use hX tags will now use P instead.
Nothing changes visually, only structurally.
Fixed some elements in Nunjucks using single quotes instead of
double quotes. They'd have worked the same, but consistency.
Added h1 title in FAQ page.
Make text for no JS warning a bit bigger, and improved the phrasing
a little bit.
2020-06-03 03:44:24 +00:00
|
|
|
if (albumids.length) {
|
2019-09-08 01:56:29 +00:00
|
|
|
db.table('albums')
|
|
|
|
.whereIn('id', albumids)
|
|
|
|
.update('editedAt', Math.floor(Date.now() / 1000))
|
|
|
|
.catch(logger.error)
|
Improved albums public page cache and more
Removed its dependency towards albums' editedAt property.
Editing album's metas (name, description, etc) will no longer update
its editedAt property.
Instead it will now ONLY be updated when adding/removing files to/from
it. Just like how it was meant to be, which was to be used to check
whether it's necessary to re-generate their downloadable ZIPs.
Albums public page cache will still be properly invalidated when
adding/removing files to/from it, as well as after editing their metas.
Added views/album-notice.njk to be used to render okay-ish notice when
an album's public page is still being generated.
I was originally thinking of using it for disabled albums as well, but
I refrained from it to reduce the possibility of disabled album IDs from
being easily scanned (as it just returns 404 now).
Removed invalidatedAt property from stats cache. Instead their caches
will immediately be nullified as they should (thus frees up memory
slightly as well).
Stats cache for albums will now only be cleared when truly necessary.
As in, adding/removing files to/from albums will no longer clear them.
Updated Nunjucks files to properly use h1, h2, h3 tags in actual
hierarchical orders.
Elements that don't need to use hX tags will now use P instead.
Nothing changes visually, only structurally.
Fixed some elements in Nunjucks using single quotes instead of
double quotes. They'd have worked the same, but consistency.
Added h1 title in FAQ page.
Make text for no JS warning a bit bigger, and improved the phrasing
a little bit.
2020-06-03 03:44:24 +00:00
|
|
|
self.invalidateAlbumsCache(albumids)
|
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
|
|
|
|
// Purge Cloudflare's cache if necessary, but do not wait
|
2020-10-30 18:12:09 +00:00
|
|
|
if (config.cloudflare.purgeCache) {
|
2019-09-08 01:56:29 +00:00
|
|
|
self.purgeCloudflareCache(unlinkeds.map(file => file.name), true, true)
|
|
|
|
.then(results => {
|
2020-10-30 18:12:09 +00:00
|
|
|
for (const result of results) {
|
|
|
|
if (result.errors.length) {
|
2019-09-08 01:56:29 +00:00
|
|
|
result.errors.forEach(error => logger.error(`[CF]: ${error}`))
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
})
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
logger.error(error)
|
|
|
|
}
|
|
|
|
|
2018-05-05 19:44:58 +00:00
|
|
|
return failed
|
2018-03-30 02:39:53 +00:00
|
|
|
}
|
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.purgeCloudflareCache = async (names, uploads, thumbs) => {
|
2020-06-26 06:48:15 +00:00
|
|
|
const errors = []
|
2020-10-30 18:12:09 +00:00
|
|
|
if (!cloudflareAuth) {
|
2020-06-26 06:48:15 +00:00
|
|
|
errors.push('Cloudflare auth is incomplete or missing')
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
|
|
|
if (!Array.isArray(names) || !names.length) {
|
2020-06-26 06:48:15 +00:00
|
|
|
errors.push('Names array is invalid or empty')
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
|
|
|
if (errors.length) {
|
2020-06-26 06:48:15 +00:00
|
|
|
return [{ success: false, files: [], errors }]
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-01-06 06:27:17 +00:00
|
|
|
|
|
|
|
let domain = config.domain
|
|
|
|
if (!uploads) domain = config.homeDomain
|
2018-05-09 09:53:27 +00:00
|
|
|
|
2019-01-18 03:40:15 +00:00
|
|
|
const thumbNames = []
|
2018-05-09 09:53:27 +00:00
|
|
|
names = names.map(name => {
|
2019-01-06 08:26:43 +00:00
|
|
|
if (uploads) {
|
|
|
|
const url = `${domain}/${name}`
|
2019-09-08 01:56:29 +00:00
|
|
|
const extname = self.extname(name)
|
2020-10-30 18:12:09 +00:00
|
|
|
if (thumbs && self.mayGenerateThumb(extname)) {
|
2019-01-18 03:40:15 +00:00
|
|
|
thumbNames.push(`${domain}/thumbs/${name.slice(0, -extname.length)}.png`)
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-01-06 08:26:43 +00:00
|
|
|
return url
|
|
|
|
} else {
|
|
|
|
return name === 'home' ? domain : `${domain}/${name}`
|
|
|
|
}
|
2018-05-09 09:53:27 +00:00
|
|
|
})
|
2020-10-11 10:32:22 +00:00
|
|
|
names.push(...thumbNames)
|
2019-01-31 09:29:34 +00:00
|
|
|
|
|
|
|
// Split array into multiple arrays with max length of 30 URLs
|
|
|
|
// https://api.cloudflare.com/#zone-purge-files-by-url
|
|
|
|
const MAX_LENGTH = 30
|
2020-12-26 09:52:07 +00:00
|
|
|
const MAX_TRIES = 3 // only for rate limit and unexpected errors
|
2019-09-08 01:56:29 +00:00
|
|
|
const chunks = []
|
2020-10-30 18:12:09 +00:00
|
|
|
while (names.length) {
|
2019-09-08 01:56:29 +00:00
|
|
|
chunks.push(names.splice(0, MAX_LENGTH))
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-01-31 09:29:34 +00:00
|
|
|
|
|
|
|
const url = `https://api.cloudflare.com/client/v4/zones/${config.cloudflare.zoneId}/purge_cache`
|
|
|
|
const results = []
|
2018-05-09 09:53:27 +00:00
|
|
|
|
2019-09-23 08:09:15 +00:00
|
|
|
await Promise.all(chunks.map(async chunk => {
|
2019-09-08 01:56:29 +00:00
|
|
|
const result = {
|
|
|
|
success: false,
|
|
|
|
files: chunk,
|
|
|
|
errors: []
|
|
|
|
}
|
2019-01-31 09:29:34 +00:00
|
|
|
|
2020-12-26 09:52:07 +00:00
|
|
|
const headers = {
|
|
|
|
'Content-Type': 'application/json'
|
|
|
|
}
|
|
|
|
if (config.cloudflare.apiToken) {
|
|
|
|
headers.Authorization = `Bearer ${config.cloudflare.apiToken}`
|
|
|
|
} else if (config.cloudflare.userServiceKey) {
|
|
|
|
headers['X-Auth-User-Service-Key'] = config.cloudflare.userServiceKey
|
|
|
|
} else if (config.cloudflare.apiKey && config.cloudflare.email) {
|
|
|
|
headers['X-Auth-Key'] = config.cloudflare.apiKey
|
|
|
|
headers['X-Auth-Email'] = config.cloudflare.email
|
|
|
|
}
|
|
|
|
|
|
|
|
for (let i = 0; i < MAX_TRIES; i++) {
|
|
|
|
const _log = message => {
|
|
|
|
let prefix = `[CF]: ${i + 1}/${MAX_TRIES}: ${path.basename(chunk[0])}`
|
|
|
|
if (chunk.length > 1) prefix += ',\u2026'
|
|
|
|
logger.log(`${prefix}: ${message}`)
|
2020-06-26 06:48:15 +00:00
|
|
|
}
|
|
|
|
|
2020-12-26 09:52:07 +00:00
|
|
|
try {
|
|
|
|
const purge = await fetch(url, {
|
|
|
|
method: 'POST',
|
|
|
|
body: JSON.stringify({ files: chunk }),
|
|
|
|
headers
|
|
|
|
})
|
|
|
|
const response = await purge.json()
|
|
|
|
|
|
|
|
const hasErrorsArray = Array.isArray(response.errors) && response.errors.length
|
|
|
|
if (hasErrorsArray) {
|
|
|
|
const rateLimit = response.errors.find(error => /rate limit/i.test(error.message))
|
|
|
|
if (rateLimit && i < MAX_TRIES - 1) {
|
|
|
|
_log(`${rateLimit.code}: ${rateLimit.message}. Retrying in a minute\u2026`)
|
|
|
|
await new Promise(resolve => setTimeout(resolve, 60000))
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
result.success = response.success
|
|
|
|
result.errors = hasErrorsArray
|
|
|
|
? response.errors.map(error => `${error.code}: ${error.message}`)
|
|
|
|
: []
|
|
|
|
} catch (error) {
|
|
|
|
const errorString = error.toString()
|
|
|
|
if (i < MAX_TRIES - 1) {
|
|
|
|
_log(`${errorString}. Retrying in 5 seconds\u2026`)
|
|
|
|
await new Promise(resolve => setTimeout(resolve, 5000))
|
|
|
|
continue
|
|
|
|
}
|
2020-06-26 06:48:15 +00:00
|
|
|
|
2020-12-26 09:52:07 +00:00
|
|
|
result.errors = [errorString]
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
|
2020-12-26 09:52:07 +00:00
|
|
|
results.push(result)
|
|
|
|
break
|
|
|
|
}
|
2019-09-23 08:09:15 +00:00
|
|
|
}))
|
2019-01-31 09:29:34 +00:00
|
|
|
|
|
|
|
return results
|
2018-05-09 09:53:27 +00:00
|
|
|
}
|
|
|
|
|
2020-05-16 14:45:14 +00:00
|
|
|
self.bulkDeleteExpired = async (dryrun, verbose) => {
|
2019-09-08 01:56:29 +00:00
|
|
|
const timestamp = Date.now() / 1000
|
2020-05-16 14:45:14 +00:00
|
|
|
const fields = ['id']
|
|
|
|
if (verbose) fields.push('name')
|
2019-09-08 01:56:29 +00:00
|
|
|
const sudo = { username: 'root' }
|
|
|
|
|
|
|
|
const result = {}
|
|
|
|
result.expired = await db.table('files')
|
|
|
|
.where('expirydate', '<=', timestamp)
|
2020-05-16 14:45:14 +00:00
|
|
|
.select(fields)
|
2019-09-08 01:56:29 +00:00
|
|
|
|
|
|
|
if (!dryrun) {
|
2020-05-16 14:45:14 +00:00
|
|
|
// Make a shallow copy
|
|
|
|
const field = fields[0]
|
|
|
|
const values = result.expired.slice().map(row => row[field])
|
2019-09-08 01:56:29 +00:00
|
|
|
result.failed = await self.bulkDeleteFromDb(field, values, sudo)
|
|
|
|
}
|
|
|
|
|
|
|
|
return result
|
2019-04-05 17:32:52 +00:00
|
|
|
}
|
|
|
|
|
2019-09-17 04:13:41 +00:00
|
|
|
self.invalidateAlbumsCache = albumids => {
|
|
|
|
for (const albumid of albumids) {
|
|
|
|
delete self.albumsCache[albumid]
|
|
|
|
delete self.albumsCache[`${albumid}-nojs`]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.invalidateStatsCache = type => {
|
2019-04-12 00:45:33 +00:00
|
|
|
if (!['albums', 'users', 'uploads'].includes(type)) return
|
2020-12-25 14:06:21 +00:00
|
|
|
statsData[type].cache = null
|
2019-04-12 00:45:33 +00:00
|
|
|
}
|
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
self.stats = async (req, res, next) => {
|
2021-01-08 03:48:08 +00:00
|
|
|
try {
|
|
|
|
const user = await self.authorize(req)
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2021-01-08 03:48:08 +00:00
|
|
|
const isadmin = perms.is(user, 'admin')
|
|
|
|
if (!isadmin) throw new ClientError('', { statusCode: 403 })
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2020-08-24 01:08:21 +00:00
|
|
|
const hrstart = process.hrtime()
|
2019-09-10 16:31:27 +00:00
|
|
|
const stats = {}
|
2020-12-25 14:06:21 +00:00
|
|
|
Object.keys(statsData).forEach(key => {
|
|
|
|
// Pre-assign object keys to fix their display order
|
|
|
|
stats[statsData[key].title] = {}
|
|
|
|
})
|
|
|
|
|
2019-09-10 16:31:27 +00:00
|
|
|
const os = await si.osInfo()
|
2020-12-25 14:06:21 +00:00
|
|
|
await Promise.all([
|
|
|
|
(async () => {
|
|
|
|
// System info
|
|
|
|
const data = statsData.system
|
|
|
|
|
|
|
|
if (!data.cache && data.generating) {
|
|
|
|
stats[data.title] = false
|
|
|
|
} else if (((Date.now() - data.generatedAt) <= 1000) || data.generating) {
|
|
|
|
// Use cache for 1000 ms (1 second)
|
|
|
|
stats[data.title] = data.cache
|
|
|
|
} else {
|
|
|
|
data.generating = true
|
|
|
|
data.generatedAt = Date.now()
|
|
|
|
|
|
|
|
const currentLoad = await si.currentLoad()
|
|
|
|
const mem = await si.mem()
|
|
|
|
const time = si.time()
|
|
|
|
const nodeUptime = process.uptime()
|
|
|
|
|
|
|
|
if (self.clamscan.instance) {
|
|
|
|
try {
|
|
|
|
self.clamscan.version = await self.clamscan.instance.get_version().then(s => s.trim())
|
|
|
|
} catch (error) {
|
|
|
|
logger.error(error)
|
|
|
|
self.clamscan.version = 'Errored when querying version.'
|
|
|
|
}
|
|
|
|
}
|
2019-09-08 01:56:29 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
stats[data.title] = {
|
|
|
|
Platform: `${os.platform} ${os.arch}`,
|
|
|
|
Distro: `${os.distro} ${os.release}`,
|
|
|
|
Kernel: os.kernel,
|
|
|
|
Scanner: self.clamscan.version || 'N/A',
|
|
|
|
'CPU Load': `${currentLoad.currentload.toFixed(1)}%`,
|
|
|
|
'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '),
|
|
|
|
'System Memory': {
|
|
|
|
value: {
|
|
|
|
used: mem.active,
|
|
|
|
total: mem.total
|
|
|
|
},
|
|
|
|
type: 'byteUsage'
|
|
|
|
},
|
|
|
|
'Memory Usage': {
|
|
|
|
value: process.memoryUsage().rss,
|
|
|
|
type: 'byte'
|
|
|
|
},
|
|
|
|
'System Uptime': {
|
|
|
|
value: time.uptime,
|
|
|
|
type: 'uptime'
|
|
|
|
},
|
|
|
|
'Node.js': `${process.versions.node}`,
|
|
|
|
'Service Uptime': {
|
|
|
|
value: Math.floor(nodeUptime),
|
|
|
|
type: 'uptime'
|
|
|
|
}
|
|
|
|
}
|
2019-09-10 16:31:27 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
// Update cache
|
|
|
|
data.cache = stats[data.title]
|
|
|
|
data.generating = false
|
|
|
|
}
|
|
|
|
})(),
|
|
|
|
(async () => {
|
|
|
|
// File systems
|
|
|
|
const data = statsData.fileSystems
|
|
|
|
|
|
|
|
if (!data.cache && data.generating) {
|
|
|
|
stats[data.title] = false
|
|
|
|
} else if (((Date.now() - data.generatedAt) <= 60000) || data.generating) {
|
|
|
|
// Use cache for 60000 ms (60 seconds)
|
|
|
|
stats[data.title] = data.cache
|
|
|
|
} else {
|
|
|
|
data.generating = true
|
|
|
|
data.generatedAt = Date.now()
|
|
|
|
|
|
|
|
stats[data.title] = {}
|
|
|
|
|
|
|
|
const fsSize = await si.fsSize()
|
|
|
|
for (const fs of fsSize) {
|
|
|
|
stats[data.title][`${fs.fs} (${fs.type}) on ${fs.mount}`] = {
|
|
|
|
value: {
|
|
|
|
total: fs.size,
|
|
|
|
used: fs.used
|
|
|
|
},
|
|
|
|
type: 'byteUsage'
|
|
|
|
}
|
|
|
|
}
|
2019-09-10 16:31:27 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
// Update cache
|
|
|
|
data.cache = stats[data.title]
|
|
|
|
data.generating = false
|
2020-11-02 11:23:23 +00:00
|
|
|
}
|
2020-12-25 14:06:21 +00:00
|
|
|
})(),
|
|
|
|
(async () => {
|
|
|
|
// Uploads
|
|
|
|
const data = statsData.uploads
|
|
|
|
|
|
|
|
if (!data.cache && data.generating) {
|
|
|
|
stats[data.title] = false
|
|
|
|
} else if (data.cache) {
|
|
|
|
// Cache will be invalidated with self.invalidateStatsCache() after any related operations
|
|
|
|
stats[data.title] = data.cache
|
|
|
|
} else {
|
|
|
|
data.generating = true
|
|
|
|
data.generatedAt = Date.now()
|
|
|
|
|
|
|
|
stats[data.title] = {
|
|
|
|
Total: 0,
|
|
|
|
Images: 0,
|
|
|
|
Videos: 0,
|
|
|
|
Others: 0,
|
2020-12-25 14:18:45 +00:00
|
|
|
Temporary: 0,
|
2020-12-25 14:06:21 +00:00
|
|
|
'Size in DB': {
|
|
|
|
value: 0,
|
|
|
|
type: 'byte'
|
|
|
|
}
|
|
|
|
}
|
2020-11-02 11:23:23 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
await Promise.all([
|
|
|
|
(async () => {
|
|
|
|
const uploads = await db.table('files')
|
|
|
|
.select('size')
|
|
|
|
stats[data.title].Total = uploads.length
|
|
|
|
stats[data.title]['Size in DB'].value = uploads.reduce((acc, upload) => acc + parseInt(upload.size), 0)
|
|
|
|
})(),
|
|
|
|
(async () => {
|
|
|
|
stats[data.title].Images = await db.table('files')
|
|
|
|
.where(function () {
|
|
|
|
for (const ext of self.imageExts) {
|
|
|
|
this.orWhere('name', 'like', `%${ext}`)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.count('id as count')
|
|
|
|
.then(rows => rows[0].count)
|
|
|
|
})(),
|
|
|
|
(async () => {
|
|
|
|
stats[data.title].Videos = await db.table('files')
|
|
|
|
.where(function () {
|
|
|
|
for (const ext of self.videoExts) {
|
|
|
|
this.orWhere('name', 'like', `%${ext}`)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.count('id as count')
|
|
|
|
.then(rows => rows[0].count)
|
2020-12-25 14:18:45 +00:00
|
|
|
})(),
|
|
|
|
(async () => {
|
|
|
|
stats[data.title].Temporary = await db.table('files')
|
|
|
|
.whereNotNull('expirydate')
|
|
|
|
.count('id as count')
|
|
|
|
.then(rows => rows[0].count)
|
2020-12-25 14:06:21 +00:00
|
|
|
})()
|
|
|
|
])
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
stats[data.title].Others = stats[data.title].Total - stats[data.title].Images - stats[data.title].Videos
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
// Update cache
|
|
|
|
data.cache = stats[data.title]
|
|
|
|
data.generating = false
|
2019-09-10 16:31:27 +00:00
|
|
|
}
|
2020-12-25 14:06:21 +00:00
|
|
|
})(),
|
|
|
|
(async () => {
|
|
|
|
// Users
|
|
|
|
const data = statsData.users
|
|
|
|
|
|
|
|
if (!data.cache && data.generating) {
|
|
|
|
stats[data.title] = false
|
|
|
|
} else if (data.cache) {
|
|
|
|
// Cache will be invalidated with self.invalidateStatsCache() after any related operations
|
|
|
|
stats[data.title] = data.cache
|
|
|
|
} else {
|
|
|
|
data.generating = true
|
|
|
|
data.generatedAt = Date.now()
|
|
|
|
|
|
|
|
stats[data.title] = {
|
|
|
|
Total: 0,
|
|
|
|
Disabled: 0
|
|
|
|
}
|
2019-09-10 16:31:27 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
const permissionKeys = Object.keys(perms.permissions).reverse()
|
|
|
|
permissionKeys.forEach(p => {
|
|
|
|
stats[data.title][p] = 0
|
2019-11-13 22:06:59 +00:00
|
|
|
})
|
2020-05-02 12:28:13 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
const users = await db.table('users')
|
|
|
|
stats[data.title].Total = users.length
|
|
|
|
for (const user of users) {
|
|
|
|
if (user.enabled === false || user.enabled === 0) {
|
|
|
|
stats[data.title].Disabled++
|
|
|
|
}
|
2019-09-10 16:31:27 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
user.permission = user.permission || 0
|
|
|
|
for (const p of permissionKeys) {
|
|
|
|
if (user.permission === perms.permissions[p]) {
|
|
|
|
stats[data.title][p]++
|
|
|
|
break
|
2019-09-10 16:31:27 +00:00
|
|
|
}
|
|
|
|
}
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2019-04-12 00:45:33 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
// Update cache
|
|
|
|
data.cache = stats[data.title]
|
|
|
|
data.generating = false
|
2020-10-30 18:12:09 +00:00
|
|
|
}
|
2020-12-25 14:06:21 +00:00
|
|
|
})(),
|
|
|
|
(async () => {
|
|
|
|
// Albums
|
|
|
|
const data = statsData.albums
|
|
|
|
|
|
|
|
if (!data.cache && data.generating) {
|
|
|
|
stats[data.title] = false
|
|
|
|
} else if (data.cache) {
|
|
|
|
// Cache will be invalidated with self.invalidateStatsCache() after any related operations
|
|
|
|
stats[data.title] = data.cache
|
|
|
|
} else {
|
|
|
|
data.generating = true
|
|
|
|
data.generatedAt = Date.now()
|
|
|
|
|
|
|
|
stats[data.title] = {
|
|
|
|
Total: 0,
|
|
|
|
Disabled: 0,
|
|
|
|
Public: 0,
|
|
|
|
Downloadable: 0,
|
|
|
|
'ZIP Generated': 0
|
2019-09-10 16:31:27 +00:00
|
|
|
}
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
const albums = await db.table('albums')
|
|
|
|
stats[data.title].Total = albums.length
|
|
|
|
const identifiers = []
|
|
|
|
for (const album of albums) {
|
|
|
|
if (!album.enabled) {
|
|
|
|
stats[data.title].Disabled++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if (album.download) stats[data.title].Downloadable++
|
|
|
|
if (album.public) stats[data.title].Public++
|
|
|
|
if (album.zipGeneratedAt) identifiers.push(album.identifier)
|
|
|
|
}
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
await Promise.all(identifiers.map(async identifier => {
|
|
|
|
try {
|
|
|
|
await paths.access(path.join(paths.zips, `${identifier}.zip`))
|
|
|
|
stats[data.title]['ZIP Generated']++
|
|
|
|
} catch (error) {
|
|
|
|
// Re-throw error
|
|
|
|
if (error.code !== 'ENOENT') throw error
|
|
|
|
}
|
|
|
|
}))
|
2019-04-05 17:32:52 +00:00
|
|
|
|
2020-12-25 14:06:21 +00:00
|
|
|
// Update cache
|
|
|
|
data.cache = stats[data.title]
|
|
|
|
data.generating = false
|
2019-09-10 16:31:27 +00:00
|
|
|
}
|
2020-12-25 14:06:21 +00:00
|
|
|
})()
|
|
|
|
])
|
2019-04-12 00:45:33 +00:00
|
|
|
|
2020-08-24 01:08:21 +00:00
|
|
|
return res.json({ success: true, stats, hrtime: process.hrtime(hrstart) })
|
2019-09-10 16:31:27 +00:00
|
|
|
} catch (error) {
|
2019-11-14 07:53:33 +00:00
|
|
|
// Reset generating state when encountering any errors
|
2020-12-25 14:06:21 +00:00
|
|
|
Object.keys(statsData).forEach(key => {
|
|
|
|
statsData[key].generating = false
|
2019-11-14 07:53:33 +00:00
|
|
|
})
|
2021-01-08 03:48:08 +00:00
|
|
|
return apiErrorsHandler(error)
|
2019-04-05 17:32:52 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-08 01:56:29 +00:00
|
|
|
module.exports = self
|