filesafe/controllers/utilsController.js

818 lines
24 KiB
JavaScript
Raw Normal View History

const { promisify } = require('util')
const AbortController = require('abort-controller')
const fastq = require('fastq')
2018-09-23 16:28:15 +00:00
const fetch = require('node-fetch')
const ffmpeg = require('fluent-ffmpeg')
const jetpack = require('fs-jetpack')
const knex = require('knex')
const MarkdownIt = require('markdown-it')
const path = require('path')
2018-12-03 07:20:13 +00:00
const sharp = require('sharp')
Updates (very important to read) Client-side CSS & JS files will now be processed with Gulp. Gulp tasks are configured in gulpfile.js file. CSS files will be optimized with postcss-preset-env, which will auto-add vendor prefixes and convert any parts necessary for browsers compatibility. Afterwards they will be minified with cssnano. JS files will be optimized with bublé, likewise for browsers compatibility. Afterwards they will be minified with terser. Unprocessed CSS & JS files will now be located at src directory, while the processed results will be located at dist directory. Due to bublé, the JS files should now be compatible up to IE 11 at the minimum. Previously the safe would not work in IE 11 due to extensive usage of template literals. Due to that as well, JS files in src directory will now extensively use arrow functions for my personal comfort (as they will be converted too). The server will use the processed files at dist directory by default. If you want to rebuild the files by your own, you can run "yarn build". Gulp is a development dependency, so make sure you have installed all development dependencies (e.i. NOT using "yarn install --production"). --- yarn lint -> gulp lint yarn build -> gulp default yarn watch -> gulp watch yarn develop -> env NODE_ENV=development yarn watch --- Fixed not being able to demote staff into normal users. /api/token/verify will no longer respond with 401 HTTP error code, unless an error occurred (which will be 500 HTTP error code). Fixed /nojs route not displaying file's original name when a duplicate is found on the server. Removed is-breeze CSS class name, in favor of Bulma's is-info. Removed custom styling from auth page, in favor of global styling. Removed all usage of style HTML attribute in favor of CSS classes. Renamed js/s/ to js/misc/. Use loading spinners on dashboard's sidebar menus. Disable all other sidebar menus when something is loading. Changed title HTML attribute of disabled control buttons in uploads & users list. Hid checkboxes and WIP controls from users list. Better error messages handling. Especially homepage will now support CF's HTTP error codes. Updated various icons. Also, added fontello config file at public/libs/fontello/config.json. This should let you edit them more easily with fontello. Use Gatsby icon for my blog's link in homepage's footer. A bunch of other improvements here & there.
2019-09-15 06:20:11 +00:00
const paths = require('./pathsController')
const perms = require('./permissionController')
const ClientError = require('./utils/ClientError')
const Constants = require('./utils/Constants')
const ServerError = require('./utils/ServerError')
const SimpleDataStore = require('./utils/SimpleDataStore')
2022-10-05 19:42:36 +00:00
const StatsManager = require('./utils/StatsManager')
2022-10-05 19:39:51 +00:00
const config = require('./utils/ConfigManager')
Updates (very important to read) Client-side CSS & JS files will now be processed with Gulp. Gulp tasks are configured in gulpfile.js file. CSS files will be optimized with postcss-preset-env, which will auto-add vendor prefixes and convert any parts necessary for browsers compatibility. Afterwards they will be minified with cssnano. JS files will be optimized with bublé, likewise for browsers compatibility. Afterwards they will be minified with terser. Unprocessed CSS & JS files will now be located at src directory, while the processed results will be located at dist directory. Due to bublé, the JS files should now be compatible up to IE 11 at the minimum. Previously the safe would not work in IE 11 due to extensive usage of template literals. Due to that as well, JS files in src directory will now extensively use arrow functions for my personal comfort (as they will be converted too). The server will use the processed files at dist directory by default. If you want to rebuild the files by your own, you can run "yarn build". Gulp is a development dependency, so make sure you have installed all development dependencies (e.i. NOT using "yarn install --production"). --- yarn lint -> gulp lint yarn build -> gulp default yarn watch -> gulp watch yarn develop -> env NODE_ENV=development yarn watch --- Fixed not being able to demote staff into normal users. /api/token/verify will no longer respond with 401 HTTP error code, unless an error occurred (which will be 500 HTTP error code). Fixed /nojs route not displaying file's original name when a duplicate is found on the server. Removed is-breeze CSS class name, in favor of Bulma's is-info. Removed custom styling from auth page, in favor of global styling. Removed all usage of style HTML attribute in favor of CSS classes. Renamed js/s/ to js/misc/. Use loading spinners on dashboard's sidebar menus. Disable all other sidebar menus when something is loading. Changed title HTML attribute of disabled control buttons in uploads & users list. Hid checkboxes and WIP controls from users list. Better error messages handling. Especially homepage will now support CF's HTTP error codes. Updated various icons. Also, added fontello config file at public/libs/fontello/config.json. This should let you edit them more easily with fontello. Use Gatsby icon for my blog's link in homepage's footer. A bunch of other improvements here & there.
2019-09-15 06:20:11 +00:00
const logger = require('./../logger')
const devmode = process.env.NODE_ENV === 'development'
const self = {
devmode,
inspect: devmode && require('util').inspect,
db: knex(config.database),
md: {
instance: new MarkdownIt({
// https://markdown-it.github.io/markdown-it/#MarkdownIt.new
html: false,
breaks: true,
linkify: true
}),
defaultRenderers: {}
},
gitHash: null,
idMaxTries: config.uploads.maxTries || 1,
stripTagsBlacklistedExts: Array.isArray(config.uploads.stripTags.blacklistExtensions)
? config.uploads.stripTags.blacklistExtensions
: [],
2020-06-06 12:43:20 +00:00
thumbsSize: config.uploads.generateThumbs.size || 200,
Updated Updated some dev dependencies. --- Gulp will now build CSS/JS files during development into dist-dev directory, to prevent IDE's Git from unnecessarily building diff's. Added dist-dev to ignore files. --- The entire config fille will now be passed to Nunjuck templates for ease of access of config values. Root domain for use in Nunjuck templates will now be parsed from config. Better page titles. Updated help message for "Uploads history order" option in homepage's config tab. Added "Load images for preview" option to homepage's config tab. Setting this to false will now prevent image uploads from loading themselves for previews. Uploads' original names in homepage's uploads history are now selectable. Min/max length for user/pass are now enforced in auth's front-end. Improved performance of album public pages. Their generated HTML pages will now be cached into memory. Unfortunately, No-JS version of their pages will be cached separately, so each album may take up to double the memory space. File names in thumbnails no longer have their full URLs as tooltips. I saw no point in that behavior. Added video icons. Homepage's uploads history will now display video icons for videos. "View thumbnail" button in Dashboard is now renamed to "Show preview". Their icons will also be changed depending on their file types. Added max length for albums' title & description. These will be enforced both in front-end and back-end. Existing albums that have surpassed the limits will not be enforced. A few other small improvements.
2019-09-17 04:13:41 +00:00
ffprobe: promisify(ffmpeg.ffprobe),
timezoneOffset: new Date().getTimezoneOffset(),
retentions: {
enabled: false,
periods: {},
default: {}
},
albumRenderStore: new SimpleDataStore({
limit: 10,
strategy: SimpleDataStore.STRATEGIES[0]
}),
contentDispositionStore: null
}
// Remember old renderer, if overridden, or proxy to default renderer
self.md.defaultRenderers.link_open = self.md.instance.renderer.rules.link_open || function (tokens, idx, options, env, that) {
return that.renderToken(tokens, idx, options)
}
// Add target="_blank" to URLs if applicable
self.md.instance.renderer.rules.link_open = function (tokens, idx, options, env, that) {
const aIndex = tokens[idx].attrIndex('target')
if (aIndex < 0) {
tokens[idx].attrPush(['target', '_blank'])
} else {
tokens[idx].attrs[aIndex][1] = '_blank'
}
return self.md.defaultRenderers.link_open(tokens, idx, options, env, that)
}
if (typeof config.uploads.retentionPeriods === 'object' &&
2022-08-23 08:16:02 +00:00
Object.keys(config.uploads.retentionPeriods).length) {
// Build a temporary index of group values
const _retentionPeriods = Object.assign({}, config.uploads.retentionPeriods)
const _groups = { _: -1 }
Object.assign(_groups, perms.permissions)
// Sanitize config values
const names = Object.keys(_groups)
for (const name of names) {
if (Array.isArray(_retentionPeriods[name]) && _retentionPeriods[name].length) {
_retentionPeriods[name] = _retentionPeriods[name]
.filter((v, i, a) => (Number.isFinite(v) && v >= 0) || v === null)
} else {
_retentionPeriods[name] = []
}
}
if (!_retentionPeriods._.length && !config.private) {
logger.error('Guests\' retention periods are missing, yet this installation is not set to private.')
process.exit(1)
}
// Create sorted array of group names based on their values
const _sorted = Object.keys(_groups)
.sort((a, b) => _groups[a] - _groups[b])
// Build retention periods array for each groups
for (let i = 0; i < _sorted.length; i++) {
const current = _sorted[i]
const _periods = [..._retentionPeriods[current]]
self.retentions.default[current] = _periods.length ? _periods[0] : null
if (i > 0) {
// Inherit retention periods of lower-valued groups
for (let j = i - 1; j >= 0; j--) {
const lower = _sorted[j]
if (_groups[lower] < _groups[current]) {
_periods.unshift(..._retentionPeriods[lower])
if (self.retentions.default[current] === null) {
self.retentions.default[current] = self.retentions.default[lower]
}
}
}
}
self.retentions.periods[current] = _periods
.filter((v, i, a) => v !== null && a.indexOf(v) === i) // re-sanitize & uniquify
.sort((a, b) => a - b) // sort from lowest to highest (zero/permanent will always be first)
// Mark the feature as enabled, if at least one group was configured
if (self.retentions.periods[current].length) {
self.retentions.enabled = true
}
}
2022-10-05 19:39:51 +00:00
} else if (Array.isArray(config.uploads.temporaryUploadAges) && config.uploads.temporaryUploadAges.length) {
self.retentions.periods._ = config.uploads.temporaryUploadAges
.filter((v, i, a) => Number.isFinite(v) && v >= 0)
self.retentions.default._ = self.retentions.periods._[0]
for (const name of Object.keys(perms.permissions)) {
self.retentions.periods[name] = self.retentions.periods._
self.retentions.default[name] = self.retentions.default._
}
self.retentions.enabled = true
}
// This helper function initiates fetch() with AbortController
// signal controller to handle per-instance global timeout.
// node-fetch's built-in timeout option resets on every redirect,
// and thus not reliable in certain cases.
self.fetch = (url, options = {}) => {
if (options.timeout === undefined) {
return fetch(url, options)
}
// Init AbortController
const abortController = new AbortController()
const timeout = setTimeout(() => {
abortController.abort()
}, options.timeout)
// Clean up options object
options.signal = abortController.signal
delete options.timeout
// Return instance with an attached Promise.finally() handler to clear timeout
return fetch(url, options)
.finally(() => {
clearTimeout(timeout)
})
}
2020-06-26 06:48:15 +00:00
const cloudflareAuth = config.cloudflare && config.cloudflare.zoneId &&
(config.cloudflare.apiToken || config.cloudflare.userServiceKey ||
(config.cloudflare.apiKey && config.cloudflare.email))
const cloudflarePurgeCacheQueue = cloudflareAuth && fastq.promise(async chunk => {
const MAX_TRIES = 3
const url = `https://api.cloudflare.com/client/v4/zones/${config.cloudflare.zoneId}/purge_cache`
const result = {
success: false,
files: chunk,
errors: []
}
const headers = {
'Content-Type': 'application/json'
}
if (config.cloudflare.apiToken) {
headers.Authorization = `Bearer ${config.cloudflare.apiToken}`
} else if (config.cloudflare.userServiceKey) {
headers['X-Auth-User-Service-Key'] = config.cloudflare.userServiceKey
} else if (config.cloudflare.apiKey && config.cloudflare.email) {
headers['X-Auth-Key'] = config.cloudflare.apiKey
headers['X-Auth-Email'] = config.cloudflare.email
}
for (let i = 0; i < MAX_TRIES; i++) {
const _log = message => {
let prefix = `[CF]: ${i + 1}/${MAX_TRIES}: ${path.basename(chunk[0])}`
if (chunk.length > 1) prefix += ',\u2026'
logger.log(`${prefix}: ${message}`)
}
const response = await fetch(url, {
method: 'POST',
body: JSON.stringify({ files: chunk }),
headers
})
.then(res => res.json())
.catch(error => error)
// If fetch errors out, instead of API responding with API errors
if (response instanceof Error) {
const errorString = response.toString()
if (i < MAX_TRIES - 1) {
_log(`${errorString}. Retrying in 5 seconds\u2026`)
await new Promise(resolve => setTimeout(resolve, 5000))
continue
}
result.errors = [errorString]
break
}
// If API reponds with API errors
const hasErrorsArray = Array.isArray(response.errors) && response.errors.length
if (hasErrorsArray) {
const rateLimit = response.errors.find(error => /rate limit/i.test(error.message))
if (rateLimit && i < MAX_TRIES - 1) {
_log(`${rateLimit.code}: ${rateLimit.message}. Retrying in a minute\u2026`)
await new Promise(resolve => setTimeout(resolve, 60000))
continue
}
}
// If succeeds or out of retries
result.success = response.success
result.errors = hasErrorsArray
? response.errors.map(error => `${error.code}: ${error.message}`)
: []
break
}
return result
}, 1) // concurrency: 1
self.mayGenerateThumb = extname => {
extname = extname.toLowerCase()
2022-10-05 19:44:31 +00:00
return (config.uploads.generateThumbs.image && Constants.IMAGE_EXTS.includes(extname)) ||
(config.uploads.generateThumbs.video && Constants.VIDEO_EXTS.includes(extname))
}
2021-01-29 16:17:56 +00:00
// Expand if necessary (should be case-insensitive)
const extPreserves = [
/\.tar\.\w+/i // tarballs
]
self.extname = (filename, lower) => {
Updates Reworked unique name generator to prevent the same unique identifier from being used if it was already used with a different extension (e.i. If a file named aBcD.jpg already exists, then files such as aBcD.png or aBcD.txt may not exist). This is mainly to deal with the fact that thumbnails are only being saved as PNG, so if the same unique name is being used by multiple image/video extensions, then only one of them will have the proper thumbnail. If you already have existing files with matching unique name but varying extensions, unfortunately you can only deal with them manually for now (either allocating new unique names or deleting them altogether). Added a new config option to filter files with no extension. Files with no extensions will no longer have their original name appended to the allocated random name (e.i. A file named "textfile" used to become something like "aBcDtextfile", where "aBcD" was the allocated random name. Now it will only just become "aBcD"). In relation to that, utils.extname() function will now always return blank string if the file name does not seem to have any extension. Though files such as '.DS_Store' (basically anything that starts with a dot) will still be accepted. Examples: .hiddenfile => .hiddenfile .hiddenfile.sh => .sh .hiddenfile.001 => .hiddenfile.001 .hiddenfile.sh.001 => .sh.001 Simplified error messages of /api/upload/finishchunks. Most, if not all, of the error responses for /api/upload* will now have HTTP status code 400 (bad request) instead of 200 (ok). I plan to generalize this for the other API routes in the future. Updated home.js to properly handle formatted error message when the response's status code is not 200 (ok). Bumped v1 version string (due to home.js).
2018-11-28 17:52:12 +00:00
// Always return blank string if the filename does not seem to have a valid extension
// Files such as .DS_Store (anything that starts with a dot, without any extension after) will still be accepted
if (!/\../.test(filename)) return ''
Updates Reworked unique name generator to prevent the same unique identifier from being used if it was already used with a different extension (e.i. If a file named aBcD.jpg already exists, then files such as aBcD.png or aBcD.txt may not exist). This is mainly to deal with the fact that thumbnails are only being saved as PNG, so if the same unique name is being used by multiple image/video extensions, then only one of them will have the proper thumbnail. If you already have existing files with matching unique name but varying extensions, unfortunately you can only deal with them manually for now (either allocating new unique names or deleting them altogether). Added a new config option to filter files with no extension. Files with no extensions will no longer have their original name appended to the allocated random name (e.i. A file named "textfile" used to become something like "aBcDtextfile", where "aBcD" was the allocated random name. Now it will only just become "aBcD"). In relation to that, utils.extname() function will now always return blank string if the file name does not seem to have any extension. Though files such as '.DS_Store' (basically anything that starts with a dot) will still be accepted. Examples: .hiddenfile => .hiddenfile .hiddenfile.sh => .sh .hiddenfile.001 => .hiddenfile.001 .hiddenfile.sh.001 => .sh.001 Simplified error messages of /api/upload/finishchunks. Most, if not all, of the error responses for /api/upload* will now have HTTP status code 400 (bad request) instead of 200 (ok). I plan to generalize this for the other API routes in the future. Updated home.js to properly handle formatted error message when the response's status code is not 200 (ok). Bumped v1 version string (due to home.js).
2018-11-28 17:52:12 +00:00
let multi = ''
let extname = ''
// check for multi-archive extensions (.001, .002, and so on)
if (/\.\d{3}$/.test(filename)) {
multi = filename.slice(filename.lastIndexOf('.') - filename.length)
filename = filename.slice(0, filename.lastIndexOf('.'))
}
// check against extensions that must be preserved
for (const extPreserve of extPreserves) {
const match = filename.match(extPreserve)
if (match && match[0]) {
extname = match[0]
break
}
}
if (!extname) {
extname = filename.slice(filename.lastIndexOf('.') - filename.length)
}
const str = extname + multi
return lower ? str.toLowerCase() : str
}
Massively overhauled uploads filtering endpoint Please consult the Help? button again to learn all the syntax changes! The prompt will now also have its width expanded! Updated dependency, knex: 0.20.13 -> 0.20.15. Added new dependency: search-query-parser. Updated all sub-dependencies. Critical? Admins-only API /users/edit will no longer return NEW password salt of the user when randomizing their password. Added page.escape() function to js/misc/utils.js. This will be used to escape input in upload filters input box. The same function used in utilsController.js. Pretty dates will now use / instead of - for date separator. This is due to the fact that date range key for filtering uploads can not accepts dates with - separator. To avoid inconsistency, we will now use / separator. Caching system of album public pages will now be disabled during development (yarn develop). Cleaned up domClick() function in js/dashboard.js. If using date or expiry range keys when filtering uploads, attach client's timezone offset to the API requets. This will be used by the server to calculate timezone differences. Success prompt when changing token will now auto-close. Removed ID column from Manage Users. Improved success prompt when editing users. This will properly list all of the edited fields at once, excluding user group change. Success message for user group change will require a bit more changes on the API endpoint, which is a bit annoying. Rebuilt client-side assets and bumped v1 version string.
2020-04-18 19:52:11 +00:00
self.escape = string => {
// MIT License
// Copyright(c) 2012-2013 TJ Holowaychuk
// Copyright(c) 2015 Andreas Lubbe
// Copyright(c) 2015 Tiancheng "Timothy" Gu
if (!string) return string
const str = String(string)
const match = /["'&<>]/.exec(str)
if (!match) return str
let escape
let html = ''
let index = 0
let lastIndex = 0
for (index = match.index; index < str.length; index++) {
switch (str.charCodeAt(index)) {
case 34: // "
escape = '&quot;'
break
case 38: // &
escape = '&amp;'
break
case 39: // '
escape = '&#39;'
break
case 60: // <
escape = '&lt;'
break
case 62: // >
escape = '&gt;'
break
default:
continue
}
if (lastIndex !== index) {
html += str.substring(lastIndex, index)
}
lastIndex = index + 1
html += escape
}
return lastIndex !== index
? html + str.substring(lastIndex, index)
: html
}
Updates (very important to read) Client-side CSS & JS files will now be processed with Gulp. Gulp tasks are configured in gulpfile.js file. CSS files will be optimized with postcss-preset-env, which will auto-add vendor prefixes and convert any parts necessary for browsers compatibility. Afterwards they will be minified with cssnano. JS files will be optimized with bublé, likewise for browsers compatibility. Afterwards they will be minified with terser. Unprocessed CSS & JS files will now be located at src directory, while the processed results will be located at dist directory. Due to bublé, the JS files should now be compatible up to IE 11 at the minimum. Previously the safe would not work in IE 11 due to extensive usage of template literals. Due to that as well, JS files in src directory will now extensively use arrow functions for my personal comfort (as they will be converted too). The server will use the processed files at dist directory by default. If you want to rebuild the files by your own, you can run "yarn build". Gulp is a development dependency, so make sure you have installed all development dependencies (e.i. NOT using "yarn install --production"). --- yarn lint -> gulp lint yarn build -> gulp default yarn watch -> gulp watch yarn develop -> env NODE_ENV=development yarn watch --- Fixed not being able to demote staff into normal users. /api/token/verify will no longer respond with 401 HTTP error code, unless an error occurred (which will be 500 HTTP error code). Fixed /nojs route not displaying file's original name when a duplicate is found on the server. Removed is-breeze CSS class name, in favor of Bulma's is-info. Removed custom styling from auth page, in favor of global styling. Removed all usage of style HTML attribute in favor of CSS classes. Renamed js/s/ to js/misc/. Use loading spinners on dashboard's sidebar menus. Disable all other sidebar menus when something is loading. Changed title HTML attribute of disabled control buttons in uploads & users list. Hid checkboxes and WIP controls from users list. Better error messages handling. Especially homepage will now support CF's HTTP error codes. Updated various icons. Also, added fontello config file at public/libs/fontello/config.json. This should let you edit them more easily with fontello. Use Gatsby icon for my blog's link in homepage's footer. A bunch of other improvements here & there.
2019-09-15 06:20:11 +00:00
self.stripIndents = string => {
if (!string) return string
Updates (very important to read) Client-side CSS & JS files will now be processed with Gulp. Gulp tasks are configured in gulpfile.js file. CSS files will be optimized with postcss-preset-env, which will auto-add vendor prefixes and convert any parts necessary for browsers compatibility. Afterwards they will be minified with cssnano. JS files will be optimized with bublé, likewise for browsers compatibility. Afterwards they will be minified with terser. Unprocessed CSS & JS files will now be located at src directory, while the processed results will be located at dist directory. Due to bublé, the JS files should now be compatible up to IE 11 at the minimum. Previously the safe would not work in IE 11 due to extensive usage of template literals. Due to that as well, JS files in src directory will now extensively use arrow functions for my personal comfort (as they will be converted too). The server will use the processed files at dist directory by default. If you want to rebuild the files by your own, you can run "yarn build". Gulp is a development dependency, so make sure you have installed all development dependencies (e.i. NOT using "yarn install --production"). --- yarn lint -> gulp lint yarn build -> gulp default yarn watch -> gulp watch yarn develop -> env NODE_ENV=development yarn watch --- Fixed not being able to demote staff into normal users. /api/token/verify will no longer respond with 401 HTTP error code, unless an error occurred (which will be 500 HTTP error code). Fixed /nojs route not displaying file's original name when a duplicate is found on the server. Removed is-breeze CSS class name, in favor of Bulma's is-info. Removed custom styling from auth page, in favor of global styling. Removed all usage of style HTML attribute in favor of CSS classes. Renamed js/s/ to js/misc/. Use loading spinners on dashboard's sidebar menus. Disable all other sidebar menus when something is loading. Changed title HTML attribute of disabled control buttons in uploads & users list. Hid checkboxes and WIP controls from users list. Better error messages handling. Especially homepage will now support CF's HTTP error codes. Updated various icons. Also, added fontello config file at public/libs/fontello/config.json. This should let you edit them more easily with fontello. Use Gatsby icon for my blog's link in homepage's footer. A bunch of other improvements here & there.
2019-09-15 06:20:11 +00:00
const result = string.replace(/^[^\S\n]+/gm, '')
const match = result.match(/^[^\S\n]*(?=\S)/gm)
const indent = match && Math.min(...match.map(el => el.length))
if (indent) {
const regexp = new RegExp(`^.{${indent}}`, 'gm')
return result.replace(regexp, '')
}
return result
}
self.mask = string => {
if (!string) return string
const max = Math.min(Math.floor(string.length / 2), 8)
const fragment = Math.floor(max / 2)
if (string.length <= fragment) {
return '*'.repeat(string.length)
} else {
return string.substring(0, fragment) +
'*'.repeat(Math.min(string.length - (fragment * 2), 4)) +
string.substring(string.length - fragment)
}
}
self.filterUniquifySqlArray = (value, index, array) => {
return value !== null &&
value !== undefined &&
value !== '' &&
array.indexOf(value) === index
}
self.unlistenEmitters = (emitters, eventName, listener) => {
emitters.forEach(emitter => {
if (!emitter) return
emitter.off(eventName, listener)
})
}
self.assertRequestType = (req, type) => {
if (!req.is(type)) {
throw new ClientError(`Request Content-Type must be ${type}.`)
}
}
self.assertJSON = async (req, res) => {
// Assert Request Content-Type
self.assertRequestType(req, 'application/json')
// Parse JSON payload
req.body = await req.json()
}
2017-10-04 00:13:38 +00:00
self.generateThumbs = async (name, extname, force) => {
extname = extname.toLowerCase()
const thumbname = path.join(paths.thumbs, name.slice(0, -extname.length) + '.png')
try {
// Check if thumbnail already exists
const stat = await jetpack.inspectAsync(thumbname)
if (stat) {
if (stat.type === 'symlink') {
// Unlink if symlink (should be symlink to the placeholder)
await jetpack.removeAsync(thumbname)
} else if (!force) {
// Continue only if it does not exist, unless forced to
return true
}
}
// Full path to input file
const input = path.join(paths.uploads, name)
// If image extension
2022-10-05 19:44:31 +00:00
if (Constants.IMAGE_EXTS.includes(extname)) {
const resizeOptions = {
2020-06-06 12:43:20 +00:00
width: self.thumbsSize,
height: self.thumbsSize,
fit: 'contain',
background: {
r: 0,
g: 0,
b: 0,
alpha: 0
}
}
const image = sharp(input)
const metadata = await image.metadata()
if (metadata.width > resizeOptions.width || metadata.height > resizeOptions.height) {
await image
.resize(resizeOptions)
.toFile(thumbname)
} else if (metadata.width === resizeOptions.width && metadata.height === resizeOptions.height) {
await image
.toFile(thumbname)
} else {
const x = resizeOptions.width - metadata.width
const y = resizeOptions.height - metadata.height
await image
.extend({
top: Math.floor(y / 2),
bottom: Math.ceil(y / 2),
left: Math.floor(x / 2),
right: Math.ceil(x / 2),
background: resizeOptions.background
})
.toFile(thumbname)
}
2022-10-05 19:44:31 +00:00
} else if (Constants.VIDEO_EXTS.includes(extname)) {
const metadata = await self.ffprobe(input)
const duration = parseInt(metadata.format.duration)
if (isNaN(duration)) {
throw new Error('File does not have valid duration metadata')
}
const videoStream = metadata.streams && metadata.streams.find(s => s.codec_type === 'video')
if (!videoStream || !videoStream.width || !videoStream.height) {
throw new Error('File does not have valid video stream metadata')
}
await new Promise((resolve, reject) => {
ffmpeg(input)
.on('error', error => reject(error))
.on('end', () => resolve())
.screenshots({
folder: paths.thumbs,
filename: name.slice(0, -extname.length) + '.png',
timemarks: [
config.uploads.generateThumbs.videoTimemark || '20%'
],
size: videoStream.width >= videoStream.height
? `${self.thumbsSize}x?`
: `?x${self.thumbsSize}`
})
})
.catch(error => error) // Error passthrough
.then(async error => {
// FFMPEG would just warn instead of exiting with errors when dealing with incomplete files
// Sometimes FFMPEG would throw errors but actually somehow succeeded in making the thumbnails
// (this could be a fallback mechanism of fluent-ffmpeg library instead)
// So instead we check if the thumbnail exists to really make sure
if (await jetpack.existsAsync(thumbname)) {
return true
} else {
throw error || new Error('FFMPEG exited with empty output file')
}
})
} else {
return false
}
} catch (error) {
logger.error(`[${name}]: generateThumbs(): ${error.toString().trim()}`)
await jetpack.removeAsync(thumbname) // try to unlink incomplete thumbs first
try {
await jetpack.symlinkAsync(paths.thumbPlaceholder, thumbname)
return true
} catch (err) {
logger.error(`[${name}]: generateThumbs(): ${err.toString().trim()}`)
return false
}
}
return true
}
self.stripTags = async (name, extname) => {
extname = extname.toLowerCase()
if (self.stripTagsBlacklistedExts.includes(extname)) return false
const fullPath = path.join(paths.uploads, name)
let tmpPath
let isError
try {
2022-10-05 19:44:31 +00:00
if (Constants.IMAGE_EXTS.includes(extname)) {
const tmpName = `tmp-${name}`
tmpPath = path.join(paths.uploads, tmpName)
await jetpack.renameAsync(fullPath, tmpName)
await sharp(tmpPath)
.toFile(fullPath)
2022-10-05 19:44:31 +00:00
} else if (config.uploads.stripTags.video && Constants.VIDEO_EXTS.includes(extname)) {
const tmpName = `tmp-${name}`
tmpPath = path.join(paths.uploads, tmpName)
await jetpack.renameAsync(fullPath, tmpName)
await new Promise((resolve, reject) => {
ffmpeg(tmpPath)
.output(fullPath)
.outputOptions([
// Experimental.
'-c copy',
'-map_metadata:g -1:g',
'-map_metadata:s:v -1:g',
'-map_metadata:s:a -1:g'
])
.on('error', error => reject(error))
.on('end', () => resolve(true))
.run()
})
} else {
return false
}
} catch (error) {
logger.error(`[${name}]: stripTags(): ${error.toString().trim()}`)
isError = true
}
if (tmpPath) {
await jetpack.removeAsync(tmpPath)
}
if (isError) {
throw new ServerError('An error occurred while stripping tags. The format may not be supported.')
}
return jetpack.inspectAsync(fullPath)
}
self.unlinkFile = async filename => {
await jetpack.removeAsync(path.join(paths.uploads, filename))
const identifier = filename.split('.')[0]
const extname = self.extname(filename, true)
2022-10-05 19:44:31 +00:00
if (Constants.IMAGE_EXTS.includes(extname) || Constants.VIDEO_EXTS.includes(extname)) {
await jetpack.removeAsync(path.join(paths.thumbs, `${identifier}.png`))
}
}
self.bulkDeleteFromDb = async (field, values = [], user, permissionBypass = false) => {
// NOTE: permissionBypass should not be set unless used by lolisafe's automated service.
if ((!user && !permissionBypass) || !['id', 'name'].includes(field) || !values.length) {
return values
2022-08-04 14:59:50 +00:00
}
// SQLITE_LIMIT_VARIABLE_NUMBER, which defaults to 999
// Read more: https://www.sqlite.org/limits.html
const MAX_VARIABLES_CHUNK_SIZE = 999
const chunks = []
while (values.length) {
chunks.push(values.splice(0, MAX_VARIABLES_CHUNK_SIZE))
}
const failed = []
const ismoderator = permissionBypass || perms.is(user, 'moderator')
try {
const unlinkeds = []
const albumids = []
// NOTE: Not wrapped within a Transaction because
// we cannot rollback files physically unlinked from the storage
await Promise.all(chunks.map(async chunk => {
const files = await self.db.table('files')
.whereIn(field, chunk)
.where(function () {
if (!ismoderator) {
2019-09-08 18:33:07 +00:00
this.where('userid', user.id)
}
})
// Push files that could not be found in db
failed.push(...chunk.filter(value => !files.find(file => file[field] === value)))
// Unlink all found files
const unlinked = []
await Promise.all(files.map(async file => {
try {
await self.unlinkFile(file.name)
unlinked.push(file)
} catch (error) {
logger.error(error)
failed.push(file[field])
}
}))
if (!unlinked.length) return
// Delete all unlinked files from db
await self.db.table('files')
.whereIn('id', unlinked.map(file => file.id))
.del()
self.invalidateStatsCache('uploads')
unlinked.forEach(file => {
// Push album ids
if (file.albumid && !albumids.includes(file.albumid)) {
albumids.push(file.albumid)
}
// Delete from Content-Disposition store if used
if (self.contentDispositionStore) {
self.contentDispositionStore.delete(file.name)
}
})
// Push unlinked files
unlinkeds.push(...unlinked)
}))
if (unlinkeds.length) {
// Update albums if necessary, but do not wait
Improved albums public page cache and more Removed its dependency towards albums' editedAt property. Editing album's metas (name, description, etc) will no longer update its editedAt property. Instead it will now ONLY be updated when adding/removing files to/from it. Just like how it was meant to be, which was to be used to check whether it's necessary to re-generate their downloadable ZIPs. Albums public page cache will still be properly invalidated when adding/removing files to/from it, as well as after editing their metas. Added views/album-notice.njk to be used to render okay-ish notice when an album's public page is still being generated. I was originally thinking of using it for disabled albums as well, but I refrained from it to reduce the possibility of disabled album IDs from being easily scanned (as it just returns 404 now). Removed invalidatedAt property from stats cache. Instead their caches will immediately be nullified as they should (thus frees up memory slightly as well). Stats cache for albums will now only be cleared when truly necessary. As in, adding/removing files to/from albums will no longer clear them. Updated Nunjucks files to properly use h1, h2, h3 tags in actual hierarchical orders. Elements that don't need to use hX tags will now use P instead. Nothing changes visually, only structurally. Fixed some elements in Nunjucks using single quotes instead of double quotes. They'd have worked the same, but consistency. Added h1 title in FAQ page. Make text for no JS warning a bit bigger, and improved the phrasing a little bit.
2020-06-03 03:44:24 +00:00
if (albumids.length) {
self.db.table('albums')
.whereIn('id', albumids)
.update('editedAt', Math.floor(Date.now() / 1000))
.catch(logger.error)
self.deleteStoredAlbumRenders(albumids)
Improved albums public page cache and more Removed its dependency towards albums' editedAt property. Editing album's metas (name, description, etc) will no longer update its editedAt property. Instead it will now ONLY be updated when adding/removing files to/from it. Just like how it was meant to be, which was to be used to check whether it's necessary to re-generate their downloadable ZIPs. Albums public page cache will still be properly invalidated when adding/removing files to/from it, as well as after editing their metas. Added views/album-notice.njk to be used to render okay-ish notice when an album's public page is still being generated. I was originally thinking of using it for disabled albums as well, but I refrained from it to reduce the possibility of disabled album IDs from being easily scanned (as it just returns 404 now). Removed invalidatedAt property from stats cache. Instead their caches will immediately be nullified as they should (thus frees up memory slightly as well). Stats cache for albums will now only be cleared when truly necessary. As in, adding/removing files to/from albums will no longer clear them. Updated Nunjucks files to properly use h1, h2, h3 tags in actual hierarchical orders. Elements that don't need to use hX tags will now use P instead. Nothing changes visually, only structurally. Fixed some elements in Nunjucks using single quotes instead of double quotes. They'd have worked the same, but consistency. Added h1 title in FAQ page. Make text for no JS warning a bit bigger, and improved the phrasing a little bit.
2020-06-03 03:44:24 +00:00
}
// Purge Cloudflare's cache if necessary, but do not wait
if (config.cloudflare.purgeCache) {
self.purgeCloudflareCache(unlinkeds.map(file => file.name), true, true)
.then(results => {
for (const result of results) {
if (result.errors.length) {
result.errors.forEach(error => logger.error(`[CF]: ${error}`))
}
}
})
}
}
} catch (error) {
logger.error(error)
}
return failed
}
self.purgeCloudflareCache = async (names, uploads, thumbs) => {
2020-06-26 06:48:15 +00:00
const errors = []
if (!cloudflareAuth) {
2020-06-26 06:48:15 +00:00
errors.push('Cloudflare auth is incomplete or missing')
}
if (!Array.isArray(names) || !names.length) {
2020-06-26 06:48:15 +00:00
errors.push('Names array is invalid or empty')
}
if (errors.length) {
2020-06-26 06:48:15 +00:00
return [{ success: false, files: [], errors }]
}
let domain = config.domain
if (!uploads) domain = config.homeDomain
const thumbNames = []
names = names.map(name => {
if (uploads) {
const url = `${domain}/${name}`
const extname = self.extname(name)
if (thumbs && self.mayGenerateThumb(extname)) {
thumbNames.push(`${domain}/thumbs/${name.slice(0, -extname.length)}.png`)
}
return url
} else {
return name === 'home' ? domain : `${domain}/${name}`
}
})
names.push(...thumbNames)
// Split array into multiple arrays with max length of 30 URLs
// https://api.cloudflare.com/#zone-purge-files-by-url
const MAX_LENGTH = 30
const chunks = []
while (names.length) {
chunks.push(names.splice(0, MAX_LENGTH))
}
const results = []
for (const chunk of chunks) {
const result = await cloudflarePurgeCacheQueue.push(chunk)
results.push(result)
}
return results
}
self.bulkDeleteExpired = async (dryrun, verbose) => {
const timestamp = Date.now() / 1000
const fields = ['id']
if (verbose) fields.push('name')
const result = {}
result.expired = await self.db.table('files')
.where('expirydate', '<=', timestamp)
.select(fields)
if (!dryrun) {
// Make a shallow copy
const field = fields[0]
const values = result.expired.slice().map(row => row[field])
// NOTE: 4th parameter set to true to bypass permission check
result.failed = await self.bulkDeleteFromDb(field, values, null, true)
if (verbose && result.failed.length) {
result.failed = result.failed
.map(failed => result.expired.find(file => file[fields[0]] === failed))
}
}
return result
}
self.deleteStoredAlbumRenders = albumids => {
Updated Updated some dev dependencies. --- Gulp will now build CSS/JS files during development into dist-dev directory, to prevent IDE's Git from unnecessarily building diff's. Added dist-dev to ignore files. --- The entire config fille will now be passed to Nunjuck templates for ease of access of config values. Root domain for use in Nunjuck templates will now be parsed from config. Better page titles. Updated help message for "Uploads history order" option in homepage's config tab. Added "Load images for preview" option to homepage's config tab. Setting this to false will now prevent image uploads from loading themselves for previews. Uploads' original names in homepage's uploads history are now selectable. Min/max length for user/pass are now enforced in auth's front-end. Improved performance of album public pages. Their generated HTML pages will now be cached into memory. Unfortunately, No-JS version of their pages will be cached separately, so each album may take up to double the memory space. File names in thumbnails no longer have their full URLs as tooltips. I saw no point in that behavior. Added video icons. Homepage's uploads history will now display video icons for videos. "View thumbnail" button in Dashboard is now renamed to "Show preview". Their icons will also be changed depending on their file types. Added max length for albums' title & description. These will be enforced both in front-end and back-end. Existing albums that have surpassed the limits will not be enforced. A few other small improvements.
2019-09-17 04:13:41 +00:00
for (const albumid of albumids) {
self.albumRenderStore.delete(`${albumid}`)
self.albumRenderStore.delete(`${albumid}-nojs`)
Updated Updated some dev dependencies. --- Gulp will now build CSS/JS files during development into dist-dev directory, to prevent IDE's Git from unnecessarily building diff's. Added dist-dev to ignore files. --- The entire config fille will now be passed to Nunjuck templates for ease of access of config values. Root domain for use in Nunjuck templates will now be parsed from config. Better page titles. Updated help message for "Uploads history order" option in homepage's config tab. Added "Load images for preview" option to homepage's config tab. Setting this to false will now prevent image uploads from loading themselves for previews. Uploads' original names in homepage's uploads history are now selectable. Min/max length for user/pass are now enforced in auth's front-end. Improved performance of album public pages. Their generated HTML pages will now be cached into memory. Unfortunately, No-JS version of their pages will be cached separately, so each album may take up to double the memory space. File names in thumbnails no longer have their full URLs as tooltips. I saw no point in that behavior. Added video icons. Homepage's uploads history will now display video icons for videos. "View thumbnail" button in Dashboard is now renamed to "Show preview". Their icons will also be changed depending on their file types. Added max length for albums' title & description. These will be enforced both in front-end and back-end. Existing albums that have surpassed the limits will not be enforced. A few other small improvements.
2019-09-17 04:13:41 +00:00
}
}
2022-10-05 19:46:49 +00:00
/** Statistics API **/
2022-10-05 19:46:49 +00:00
self.invalidateStatsCache = StatsManager.invalidateStatsCache
self.buildStatsPayload = name => {
return {
...((StatsManager.cachedStats[name] && StatsManager.cachedStats[name].cache) || {}),
meta: {
key: name,
...(StatsManager.cachedStats[name]
? {
cached: Boolean(StatsManager.cachedStats[name].cache),
generatedOn: StatsManager.cachedStats[name].generatedOn || 0,
maxAge: typeof StatsManager.statGenerators[name].maxAge === 'number'
? StatsManager.statGenerators[name].maxAge
: null
}
: {
cached: false
}),
type: StatsManager.Type.HIDDEN
}
}
}
2022-10-05 19:46:49 +00:00
self.stats = async (req, res) => {
const isadmin = perms.is(req.locals.user, 'admin')
if (!isadmin) {
return res.status(403).end()
}
const hrstart = process.hrtime()
await StatsManager.generateStats(self.db)
// Ensures object payload has its keys matching the required ordering
const stats = StatsManager.statNames.reduce((acc, name) => {
const title = StatsManager.statGenerators[name].title
acc[title] = self.buildStatsPayload(name)
return acc
}, {})
return res.json({ success: true, stats, hrtime: process.hrtime(hrstart) })
}
self.statsCategory = async (req, res) => {
const isadmin = perms.is(req.locals.user, 'admin')
if (!isadmin) {
return res.status(403).end()
}
const category = req.path_parameters && req.path_parameters.category
if (!category || !StatsManager.statNames.includes(category)) {
throw new ClientError('Bad request.')
}
const hrstart = process.hrtime()
// Generate required stats category, forced
await StatsManager.generateStats(self.db, [category], true)
const title = StatsManager.statGenerators[category].title
const stats = {
[title]: self.buildStatsPayload(category)
}
return res.json({ success: true, stats, hrtime: process.hrtime(hrstart) })
}
module.exports = self