const config = require('./../config') const crypto = require('crypto') const db = require('knex')(config.database) const fetch = require('node-fetch') const fs = require('fs') const multer = require('multer') const path = require('path') const perms = require('./permissionController') const randomstring = require('randomstring') const utils = require('./utilsController') const uploadsController = {} const maxTries = config.uploads.maxTries || 1 const uploadsDir = path.join(__dirname, '..', config.uploads.folder) const chunkedUploads = Boolean(config.uploads.chunkSize) const chunksDir = path.join(uploadsDir, 'chunks') const maxSize = config.uploads.maxSize const maxSizeBytes = parseInt(maxSize) * 1000000 const urlMaxSizeBytes = parseInt(config.uploads.urlMaxSize) * 1000000 const storage = multer.diskStorage({ destination (req, file, cb) { // If chunked uploads is disabled or the uploaded file is not a chunk if (!chunkedUploads || (req.body.uuid === undefined && req.body.chunkindex === undefined)) { return cb(null, uploadsDir) } const uuidDir = path.join(chunksDir, req.body.uuid) fs.access(uuidDir, error => { if (!error) { return cb(null, uuidDir) } fs.mkdir(uuidDir, error => { if (!error) { return cb(null, uuidDir) } console.error(error) // eslint-disable-next-line standard/no-callback-literal return cb('Could not process the chunked upload. Try again?') }) }) }, filename (req, file, cb) { // If chunked uploads is disabled or the uploaded file is not a chunk if (!chunkedUploads || (req.body.uuid === undefined && req.body.chunkindex === undefined)) { const extension = utils.extname(file.originalname) const length = uploadsController.getFileNameLength(req) return uploadsController.getUniqueRandomName(length, extension, req.app.get('uploads-set')) .then(name => cb(null, name)) .catch(error => cb(error)) } // index.extension (e.i. 0, 1, ..., n - will prepend zeros depending on the amount of chunks) const digits = req.body.totalchunkcount !== undefined ? String(req.body.totalchunkcount - 1).length : 1 const zeros = new Array(digits + 1).join('0') const name = (zeros + req.body.chunkindex).slice(-digits) return cb(null, name) } }) const upload = multer({ storage, limits: { fileSize: maxSizeBytes }, fileFilter (req, file, cb) { const extname = utils.extname(file.originalname) if (uploadsController.isExtensionFiltered(extname)) { // eslint-disable-next-line standard/no-callback-literal return cb(`${extname ? `${extname.substr(1).toUpperCase()} files` : 'Files with no extension'} are not permitted.`) } // Re-map Dropzone keys so people can manually use the API without prepending 'dz' for (const key in req.body) { if (!/^dz/.test(key)) { continue } req.body[key.replace(/^dz/, '')] = req.body[key] delete req.body[key] } if (req.body.chunkindex) { if (chunkedUploads && parseInt(req.body.totalfilesize) > maxSizeBytes) { // This will not be true if "totalfilesize" key does not exist, since "NaN > number" is false. // eslint-disable-next-line standard/no-callback-literal return cb('Chunk error occurred. Total file size is larger than the maximum file size.') } else if (!chunkedUploads) { // eslint-disable-next-line standard/no-callback-literal return cb('Chunked uploads are disabled at the moment.') } } return cb(null, true) } }).array('files[]') uploadsController.isExtensionFiltered = extname => { if (!extname && config.filterNoExtension) { return true } // If there are extensions that have to be filtered if (extname && config.extensionsFilter && config.extensionsFilter.length) { const match = config.extensionsFilter.some(extension => extname === extension.toLowerCase()) if ((config.filterBlacklist && match) || (!config.filterBlacklist && !match)) { return true } } return false } uploadsController.getFileNameLength = req => { // If the user has a preferred file length, make sure it is within the allowed range if (req.headers.filelength) { return Math.min(Math.max(req.headers.filelength, config.uploads.fileLength.min), config.uploads.fileLength.max) } // Let's default it to 32 characters when config key is falsy return config.uploads.fileLength.default || 32 } uploadsController.getUniqueRandomName = (length, extension, set) => { return new Promise((resolve, reject) => { const access = i => { const identifier = randomstring.generate(length) if (config.uploads.cacheFileIdentifiers) { // Check whether the identifier is already used in cache if (set.has(identifier)) { console.log(`Identifier ${identifier} is already in use (${++i}/${maxTries}).`) if (i < maxTries) { return access(i) } // eslint-disable-next-line prefer-promise-reject-errors return reject('Sorry, we could not allocate a unique random name. Try again?') } set.add(identifier) // console.log(`Added ${identifier} to identifiers cache`) return resolve(identifier + extension) } else { // Less stricter collision check, as in the same identifier // can be used by multiple different extensions const name = identifier + extension fs.access(path.join(uploadsDir, name), error => { if (error) { return resolve(name) } console.log(`A file named ${name} already exists (${++i}/${maxTries}).`) if (i < maxTries) { return access(i) } // eslint-disable-next-line prefer-promise-reject-errors return reject('Sorry, we could not allocate a unique random name. Try again?') }) } } access(0) }) } uploadsController.upload = async (req, res, next) => { let user if (config.private === true) { user = await utils.authorize(req, res) if (!user) { return } } else if (req.headers.token) { user = await db.table('users').where('token', req.headers.token).first() } if (user && (user.enabled === false || user.enabled === 0)) { return res.json({ success: false, description: 'This account has been disabled.' }) } if (user && user.fileLength && !req.headers.filelength) { req.headers.filelength = user.fileLength } let albumid = parseInt(req.headers.albumid || req.params.albumid) if (isNaN(albumid)) { albumid = null } if (req.body.urls) { return uploadsController.actuallyUploadByUrl(req, res, user, albumid) } else { return uploadsController.actuallyUpload(req, res, user, albumid) } } uploadsController.actuallyUpload = async (req, res, user, albumid) => { const erred = error => { const isError = error instanceof Error if (isError) { console.error(error) } res.status(400).json({ success: false, description: isError ? error.toString() : error }) } upload(req, res, async error => { if (error) { const expected = [ 'LIMIT_FILE_SIZE', 'LIMIT_UNEXPECTED_FILE' ] if (expected.includes(error.code)) { return erred(error.toString()) } return erred(error) } if (!req.files || !req.files.length) { return erred('No files.') } // If chunked uploads is enabled and the uploaded file is a chunk, then just say that it was a success if (chunkedUploads && req.body.uuid) { return res.json({ success: true }) } const infoMap = req.files.map(file => { file.albumid = albumid return { path: path.join(__dirname, '..', config.uploads.folder, file.filename), data: file } }) if (config.uploads.scan && config.uploads.scan.enabled) { const scan = await uploadsController.scanFiles(req, infoMap) if (scan) { return erred(scan) } } const result = await uploadsController.formatInfoMap(req, res, user, infoMap) .catch(erred) if (!result) { return } uploadsController.processFilesForDisplay(req, res, result.files, result.existingFiles) }) } uploadsController.actuallyUploadByUrl = async (req, res, user, albumid) => { const erred = error => { const isError = error instanceof Error if (isError) { console.error(error) } res.status(400).json({ success: false, description: isError ? error.toString() : error }) } if (!config.uploads.urlMaxSize) { return erred('Upload by URLs is disabled at the moment.') } const urls = req.body.urls if (!urls || !(urls instanceof Array)) { return erred('Missing "urls" property (Array).') } // DuckDuckGo's proxy if (config.uploads.urlDuckDuckGoProxy) { return erred('URL uploads unavailable. Please contact the site owner.') // urls = urls.map(url => `https://proxy.duckduckgo.com/iu/?u=${encodeURIComponent(url)}&f=1`) } let iteration = 0 const infoMap = [] for (const url of urls) { const original = path.basename(url).split(/[?#]/)[0] const extension = utils.extname(original) if (uploadsController.isExtensionFiltered(extension)) { return erred(`${extension.substr(1).toUpperCase()} files are not permitted due to security reasons.`) } try { const fetchHead = await fetch(url, { method: 'HEAD' }) if (fetchHead.status !== 200) { return erred(`${fetchHead.status} ${fetchHead.statusText}`) } const headers = fetchHead.headers const size = parseInt(headers.get('content-length')) if (isNaN(size)) { return erred('URLs with missing Content-Length HTTP header are not supported.') } if (size > urlMaxSizeBytes) { return erred('File too large.') } // limit max response body size with content-length const fetchFile = await fetch(url, { size }) if (fetchFile.status !== 200) { return erred(`${fetchHead.status} ${fetchHead.statusText}`) } const file = await fetchFile.buffer() const length = uploadsController.getFileNameLength(req) const name = await uploadsController.getUniqueRandomName(length, extension, req.app.get('uploads-set')) const destination = path.join(uploadsDir, name) fs.writeFile(destination, file, async error => { if (error) { return erred(error) } const data = { filename: name, originalname: original, mimetype: headers.get('content-type').split(';')[0] || '', size, albumid } infoMap.push({ path: destination, data }) iteration++ if (iteration === urls.length) { if (config.uploads.scan && config.uploads.scan.enabled) { const scan = await uploadsController.scanFiles(req, infoMap) if (scan) { return erred(scan) } } const result = await uploadsController.formatInfoMap(req, res, user, infoMap) .catch(erred) if (!result) { return } uploadsController.processFilesForDisplay(req, res, result.files, result.existingFiles) } }) } catch (error) { erred(error) } } } uploadsController.finishChunks = async (req, res, next) => { if (!chunkedUploads) { return res.json({ success: false, description: 'Chunked upload is disabled at the moment.' }) } let user if (config.private === true) { user = await utils.authorize(req, res) if (!user) { return } } else if (req.headers.token) { user = await db.table('users').where('token', req.headers.token).first() } if (user && (user.enabled === false || user.enabled === 0)) { return res.json({ success: false, description: 'This account has been disabled.' }) } if (user && user.fileLength && !req.headers.filelength) { req.headers.filelength = user.fileLength } let albumid = parseInt(req.headers.albumid || req.params.albumid) if (isNaN(albumid)) { albumid = null } return uploadsController.actuallyFinishChunks(req, res, user, albumid) } uploadsController.actuallyFinishChunks = async (req, res, user, albumid) => { const erred = error => { const isError = error instanceof Error if (isError) { console.error(error) } res.status(400).json({ success: false, description: isError ? error.toString() : error }) } const files = req.body.files if (!files || !(files instanceof Array) || !files.length) { return erred('Invalid "files" property (Array).') } let iteration = 0 const infoMap = [] for (const file of files) { if (!file.uuid || typeof file.uuid !== 'string') { return erred('Invalid "uuid" property (string).') } if (typeof file.count !== 'number' || file.count < 1) { return erred('Invalid "count" property (number).') } const uuidDir = path.join(chunksDir, file.uuid) fs.readdir(uuidDir, async (error, chunkNames) => { if (error) { if (error.code === 'ENOENT') { return erred('UUID is not being used.') } return erred(error) } if (file.count < chunkNames.length) { return erred('Chunks count mismatch.') } const extension = typeof file.original === 'string' ? utils.extname(file.original) : '' if (uploadsController.isExtensionFiltered(extension)) { return erred(`${extension.substr(1).toUpperCase()} files are not permitted due to security reasons.`) } const length = uploadsController.getFileNameLength(req) const name = await uploadsController.getUniqueRandomName(length, extension, req.app.get('uploads-set')) .catch(erred) if (!name) { return } const destination = path.join(uploadsDir, name) // Sort chunk names chunkNames.sort() // Get total chunks size const chunksTotalSize = await uploadsController.getTotalSize(uuidDir, chunkNames) .catch(erred) if (typeof chunksTotalSize !== 'number') { return } if (chunksTotalSize > maxSizeBytes) { // Delete all chunks and remove chunks dir const chunksCleaned = await uploadsController.cleanUpChunks(uuidDir, chunkNames) .catch(erred) if (!chunksCleaned) { return } return erred(`Total chunks size is bigger than ${maxSize}.`) } // Append all chunks const destFileStream = fs.createWriteStream(destination, { flags: 'a' }) const chunksAppended = await uploadsController.appendToStream(destFileStream, uuidDir, chunkNames) .catch(erred) if (!chunksAppended) { return } // Delete all chunks and remove chunks dir const chunksCleaned = await uploadsController.cleanUpChunks(uuidDir, chunkNames) .catch(erred) if (!chunksCleaned) { return } const data = { filename: name, originalname: file.original || '', mimetype: file.type || '', size: file.size || 0 } data.albumid = parseInt(file.albumid) if (isNaN(data.albumid)) { data.albumid = albumid } infoMap.push({ path: destination, data }) iteration++ if (iteration === files.length) { if (config.uploads.scan && config.uploads.scan.enabled) { const scan = await uploadsController.scanFiles(req, infoMap) if (scan) { return erred(scan) } } const result = await uploadsController.formatInfoMap(req, res, user, infoMap) .catch(erred) if (!result) { return } uploadsController.processFilesForDisplay(req, res, result.files, result.existingFiles) } }) } } uploadsController.getTotalSize = (uuidDir, chunkNames) => { return new Promise((resolve, reject) => { let size = 0 const stat = i => { if (i === chunkNames.length) { return resolve(size) } fs.stat(path.join(uuidDir, chunkNames[i]), (error, stats) => { if (error) { return reject(error) } size += stats.size stat(i + 1) }) } stat(0) }) } uploadsController.appendToStream = (destFileStream, uuidDr, chunkNames) => { return new Promise((resolve, reject) => { const append = i => { if (i === chunkNames.length) { destFileStream.end() return resolve(true) } fs.createReadStream(path.join(uuidDr, chunkNames[i])) .on('end', () => { append(i + 1) }) .on('error', error => { console.error(error) destFileStream.end() return reject(error) }) .pipe(destFileStream, { end: false }) } append(0) }) } uploadsController.cleanUpChunks = (uuidDir, chunkNames) => { return new Promise(async (resolve, reject) => { await Promise.all(chunkNames.map(chunkName => { return new Promise((resolve, reject) => { const chunkPath = path.join(uuidDir, chunkName) fs.unlink(chunkPath, error => { if (error && error.code !== 'ENOENT') { console.error(error) return reject(error) } resolve() }) }) })).catch(reject) fs.rmdir(uuidDir, error => { if (error) { return reject(error) } resolve(true) }) }) } uploadsController.formatInfoMap = (req, res, user, infoMap) => { return new Promise(async resolve => { let iteration = 0 const files = [] const existingFiles = [] const albumsAuthorized = {} for (const info of infoMap) { // Check if the file exists by checking hash and size const hash = crypto.createHash('md5') const stream = fs.createReadStream(info.path) stream.on('data', data => { hash.update(data, 'utf8') }) stream.on('end', async () => { const fileHash = hash.digest('hex') const dbFile = await db.table('files') .where(function () { if (user === undefined) { this.whereNull('userid') } else { this.where('userid', user.id) } }) .where({ hash: fileHash, size: info.data.size }) .first() if (!dbFile) { if (info.data.albumid && albumsAuthorized[info.data.albumid] === undefined) { const authorized = await db.table('albums') .where({ id: info.data.albumid, userid: user.id }) .first() albumsAuthorized[info.data.albumid] = Boolean(authorized) } files.push({ name: info.data.filename, original: info.data.originalname, type: info.data.mimetype, size: info.data.size, hash: fileHash, ip: req.ip, albumid: albumsAuthorized[info.data.albumid] ? info.data.albumid : null, userid: user !== undefined ? user.id : null, timestamp: Math.floor(Date.now() / 1000) }) } else { utils.deleteFile(info.data.filename).catch(console.error) const set = req.app.get('uploads-set') if (set) { const identifier = info.data.filename.split('.')[0] set.delete(identifier) // console.log(`Removed ${identifier} from identifiers cache (formatInfoMap)`) } existingFiles.push(dbFile) } iteration++ if (iteration === infoMap.length) { resolve({ files, existingFiles }) } }) } }) } uploadsController.scanFiles = (req, infoMap) => { return new Promise(async (resolve, reject) => { const scanner = req.app.get('clam-scanner') let iteration = 0 for (const info of infoMap) { scanner.scanFile(info.path).then(reply => { if (!reply.includes('OK') || reply.includes('FOUND')) { // eslint-disable-next-line no-control-regex const virus = reply.replace(/^stream: /, '').replace(/ FOUND\u0000$/, '') console.log(`ClamAV: ${info.data.filename}: ${virus} FOUND.`) return resolve(virus) } iteration++ if (iteration === infoMap.length) { resolve(null) } }).catch(reject) } }).then(virus => { if (!virus) { return false } // If there is at least one dirty file, then delete all files const set = req.app.get('uploads-set') infoMap.forEach(info => { utils.deleteFile(info.data.filename).catch(console.error) if (set) { const identifier = info.data.filename.split('.')[0] set.delete(identifier) // console.log(`Removed ${identifier} from identifiers cache (formatInfoMap)`) } }) // Unfortunately, we will only be returning name of the first virus // even if the current session was made up by multiple virus types return `Virus detected: ${virus}.` }).catch(error => { console.error(`ClamAV: ${error.toString()}.`) return `ClamAV: ${error.code}, please contact site owner.` }) } uploadsController.processFilesForDisplay = async (req, res, files, existingFiles) => { const responseFiles = [] if (files.length) { // Insert new files to DB await db.table('files').insert(files) for (const file of files) { responseFiles.push(file) } } if (existingFiles.length) { for (const file of existingFiles) { responseFiles.push(file) } } // We send response first before generating thumbnails and updating album timestamps res.json({ success: true, files: responseFiles.map(file => { return { name: file.name, size: file.size, url: `${config.domain}/${file.name}` } }) }) const albumids = [] for (const file of files) { if (file.albumid && !albumids.includes(file.albumid)) { albumids.push(file.albumid) } if (utils.mayGenerateThumb(utils.extname(file.name))) { utils.generateThumbs(file.name) } } if (albumids.length) { await db.table('albums') .whereIn('id', albumids) .update('editedAt', Math.floor(Date.now() / 1000)) .catch(console.error) } } uploadsController.delete = async (req, res) => { const id = parseInt(req.body.id) req.body.field = 'id' req.body.values = isNaN(id) ? undefined : [id] delete req.body.id return uploadsController.bulkDelete(req, res) } uploadsController.bulkDelete = async (req, res) => { const user = await utils.authorize(req, res) if (!user) { return } const field = req.body.field || 'id' const values = req.body.values if (!values || values.constructor.name !== 'Array' || !values.length) { return res.json({ success: false, description: 'No array of files specified.' }) } const failed = await utils.bulkDeleteFiles(field, values, user, req.app.get('uploads-set')) if (failed.length < values.length) { return res.json({ success: true, failed }) } return res.json({ success: false, description: 'Could not delete any files.' }) } uploadsController.list = async (req, res) => { const user = await utils.authorize(req, res) if (!user) { return } let offset = req.params.page if (offset === undefined) { offset = 0 } // Headers is string-only, this seem to be the safest and lightest const all = req.headers.all === '1' const ismoderator = perms.is(user, 'moderator') if (all && !ismoderator) { return res.json(403) } const files = await db.table('files') .where(function () { if (req.params.id === undefined) { this.where('id', '<>', '') } else { this.where('albumid', req.params.id) } }) .where(function () { if (!all || !ismoderator) { this.where('userid', user.id) } }) .orderBy('id', 'DESC') .limit(25) .offset(25 * offset) .select('id', 'albumid', 'timestamp', 'name', 'userid', 'size') const albums = await db.table('albums') .where(function () { this.where('enabled', 1) if (!all || !ismoderator) { this.where('userid', user.id) } }) const basedomain = config.domain const userids = [] for (const file of files) { file.file = `${basedomain}/${file.name}` file.album = '' if (file.albumid !== undefined) { for (const album of albums) { if (file.albumid === album.id) { file.album = album.name } } } // Only push usernames if we are a moderator if (all && ismoderator) { if (file.userid !== undefined && file.userid !== null && file.userid !== '') { userids.push(file.userid) } } file.extname = utils.extname(file.name) if (utils.mayGenerateThumb(file.extname)) { file.thumb = `${basedomain}/thumbs/${file.name.slice(0, -file.extname.length)}.png` } } // If we are a normal user, send response if (!ismoderator) { return res.json({ success: true, files }) } // If we are a moderator but there are no uploads attached to a user, send response if (userids.length === 0) { return res.json({ success: true, files }) } const users = await db.table('users').whereIn('id', userids) for (const dbUser of users) { for (const file of files) { if (file.userid === dbUser.id) { file.username = dbUser.username } } } return res.json({ success: true, files }) } module.exports = uploadsController