filesafe/controllers/uploadController.js
Bobby Wibowo 53789a20c2
Updated uploadController.js
DuckDuckGo's proxy is no longer supported as it stops reporting Content-Length header, which is crucial so that the safe could predict the actual file size before downloading it.

If you have it enabled in your config file, it will now close the safe with error code 1. You can either disable url uploads completely or just disable duckduckgo's proxy (though I believe not many will choose the latter as to begin with it was implemented to hide origin IP).
2018-12-09 00:55:04 +07:00

765 lines
25 KiB
JavaScript

const config = require('./../config')
const crypto = require('crypto')
const db = require('knex')(config.database)
const fetch = require('node-fetch')
const fs = require('fs')
const multer = require('multer')
const path = require('path')
const perms = require('./permissionController')
const randomstring = require('randomstring')
const utils = require('./utilsController')
const uploadsController = {}
const maxTries = config.uploads.maxTries || 1
const uploadsDir = path.join(__dirname, '..', config.uploads.folder)
const chunkedUploads = Boolean(config.uploads.chunkSize)
const chunksDir = path.join(uploadsDir, 'chunks')
const maxSize = config.uploads.maxSize
const maxSizeBytes = parseInt(maxSize) * 1000000
const urlMaxSizeBytes = parseInt(config.uploads.urlMaxSize) * 1000000
const storage = multer.diskStorage({
destination (req, file, cb) {
// If chunked uploads is disabled or the uploaded file is not a chunk
if (!chunkedUploads || (req.body.uuid === undefined && req.body.chunkindex === undefined)) {
return cb(null, uploadsDir)
}
const uuidDir = path.join(chunksDir, req.body.uuid)
fs.access(uuidDir, error => {
if (!error) { return cb(null, uuidDir) }
fs.mkdir(uuidDir, error => {
if (!error) { return cb(null, uuidDir) }
console.error(error)
// eslint-disable-next-line standard/no-callback-literal
return cb('Could not process the chunked upload. Try again?')
})
})
},
filename (req, file, cb) {
// If chunked uploads is disabled or the uploaded file is not a chunk
if (!chunkedUploads || (req.body.uuid === undefined && req.body.chunkindex === undefined)) {
const extension = utils.extname(file.originalname)
const length = uploadsController.getFileNameLength(req)
return uploadsController.getUniqueRandomName(length, extension, req.app.get('uploads-set'))
.then(name => cb(null, name))
.catch(error => cb(error))
}
// index.extension (e.i. 0, 1, ..., n - will prepend zeros depending on the amount of chunks)
const digits = req.body.totalchunkcount !== undefined ? String(req.body.totalchunkcount - 1).length : 1
const zeros = new Array(digits + 1).join('0')
const name = (zeros + req.body.chunkindex).slice(-digits)
return cb(null, name)
}
})
const upload = multer({
storage,
limits: {
fileSize: maxSizeBytes
},
fileFilter (req, file, cb) {
const extname = utils.extname(file.originalname)
if (uploadsController.isExtensionFiltered(extname)) {
// eslint-disable-next-line standard/no-callback-literal
return cb(`${extname ? `${extname.substr(1).toUpperCase()} files` : 'Files with no extension'} are not permitted.`)
}
// Re-map Dropzone keys so people can manually use the API without prepending 'dz'
for (const key in req.body) {
if (!/^dz/.test(key)) { continue }
req.body[key.replace(/^dz/, '')] = req.body[key]
delete req.body[key]
}
if (req.body.chunkindex) {
if (chunkedUploads && parseInt(req.body.totalfilesize) > maxSizeBytes) {
// This will not be true if "totalfilesize" key does not exist, since "NaN > number" is false.
// eslint-disable-next-line standard/no-callback-literal
return cb('Chunk error occurred. Total file size is larger than the maximum file size.')
} else if (!chunkedUploads) {
// eslint-disable-next-line standard/no-callback-literal
return cb('Chunked uploads are disabled at the moment.')
}
}
return cb(null, true)
}
}).array('files[]')
uploadsController.isExtensionFiltered = extname => {
if (!extname && config.filterNoExtension) { return true }
// If there are extensions that have to be filtered
if (extname && config.extensionsFilter && config.extensionsFilter.length) {
const match = config.extensionsFilter.some(extension => extname === extension.toLowerCase())
if ((config.filterBlacklist && match) || (!config.filterBlacklist && !match)) {
return true
}
}
return false
}
uploadsController.getFileNameLength = req => {
// If the user has a preferred file length, make sure it is within the allowed range
if (req.headers.filelength) {
return Math.min(Math.max(req.headers.filelength, config.uploads.fileLength.min), config.uploads.fileLength.max)
}
// Let's default it to 32 characters when config key is falsy
return config.uploads.fileLength.default || 32
}
uploadsController.getUniqueRandomName = (length, extension, set) => {
return new Promise((resolve, reject) => {
const access = i => {
const identifier = randomstring.generate(length)
if (config.uploads.cacheFileIdentifiers) {
// Check whether the identifier is already used in cache
if (set.has(identifier)) {
console.log(`Identifier ${identifier} is already in use (${++i}/${maxTries}).`)
if (i < maxTries) { return access(i) }
// eslint-disable-next-line prefer-promise-reject-errors
return reject('Sorry, we could not allocate a unique random name. Try again?')
}
set.add(identifier)
// console.log(`Added ${identifier} to identifiers cache`)
return resolve(identifier + extension)
} else {
// Less stricter collision check, as in the same identifier
// can be used by multiple different extensions
const name = identifier + extension
fs.access(path.join(uploadsDir, name), error => {
if (error) { return resolve(name) }
console.log(`A file named ${name} already exists (${++i}/${maxTries}).`)
if (i < maxTries) { return access(i) }
// eslint-disable-next-line prefer-promise-reject-errors
return reject('Sorry, we could not allocate a unique random name. Try again?')
})
}
}
access(0)
})
}
uploadsController.upload = async (req, res, next) => {
let user
if (config.private === true) {
user = await utils.authorize(req, res)
if (!user) { return }
} else if (req.headers.token) {
user = await db.table('users').where('token', req.headers.token).first()
}
if (user && (user.enabled === false || user.enabled === 0)) {
return res.json({ success: false, description: 'This account has been disabled.' })
}
if (user && user.fileLength && !req.headers.filelength) {
req.headers.filelength = user.fileLength
}
let albumid = parseInt(req.headers.albumid || req.params.albumid)
if (isNaN(albumid)) { albumid = null }
if (req.body.urls) {
return uploadsController.actuallyUploadByUrl(req, res, user, albumid)
} else {
return uploadsController.actuallyUpload(req, res, user, albumid)
}
}
uploadsController.actuallyUpload = async (req, res, user, albumid) => {
const erred = error => {
const isError = error instanceof Error
if (isError) { console.error(error) }
res.status(400).json({
success: false,
description: isError ? error.toString() : error
})
}
upload(req, res, async error => {
if (error) {
const expected = [
'LIMIT_FILE_SIZE',
'LIMIT_UNEXPECTED_FILE'
]
if (expected.includes(error.code)) { return erred(error.toString()) }
return erred(error)
}
if (!req.files || !req.files.length) { return erred('No files.') }
// If chunked uploads is enabled and the uploaded file is a chunk, then just say that it was a success
if (chunkedUploads && req.body.uuid) { return res.json({ success: true }) }
const infoMap = req.files.map(file => {
file.albumid = albumid
return {
path: path.join(__dirname, '..', config.uploads.folder, file.filename),
data: file
}
})
if (config.uploads.scan && config.uploads.scan.enabled) {
const scan = await uploadsController.scanFiles(req, infoMap)
if (scan) { return erred(scan) }
}
const result = await uploadsController.formatInfoMap(req, res, user, infoMap)
.catch(erred)
if (!result) { return }
uploadsController.processFilesForDisplay(req, res, result.files, result.existingFiles)
})
}
uploadsController.actuallyUploadByUrl = async (req, res, user, albumid) => {
const erred = error => {
const isError = error instanceof Error
if (isError) { console.error(error) }
res.status(400).json({
success: false,
description: isError ? error.toString() : error
})
}
if (!config.uploads.urlMaxSize) { return erred('Upload by URLs is disabled at the moment.') }
const urls = req.body.urls
if (!urls || !(urls instanceof Array)) { return erred('Missing "urls" property (Array).') }
// DuckDuckGo's proxy
if (config.uploads.urlDuckDuckGoProxy) {
return erred('URL uploads unavailable. Please contact the site owner.')
// urls = urls.map(url => `https://proxy.duckduckgo.com/iu/?u=${encodeURIComponent(url)}&f=1`)
}
let iteration = 0
const infoMap = []
for (const url of urls) {
const original = path.basename(url).split(/[?#]/)[0]
const extension = utils.extname(original)
if (uploadsController.isExtensionFiltered(extension)) {
return erred(`${extension.substr(1).toUpperCase()} files are not permitted due to security reasons.`)
}
try {
const fetchHead = await fetch(url, { method: 'HEAD' })
if (fetchHead.status !== 200) {
return erred(`${fetchHead.status} ${fetchHead.statusText}`)
}
const headers = fetchHead.headers
const size = parseInt(headers.get('content-length'))
if (isNaN(size)) {
return erred('URLs with missing Content-Length HTTP header are not supported.')
}
if (size > urlMaxSizeBytes) {
return erred('File too large.')
}
const fetchFile = await fetch(url)
if (fetchFile.status !== 200) {
return erred(`${fetchHead.status} ${fetchHead.statusText}`)
}
const file = await fetchFile.buffer()
const length = uploadsController.getFileNameLength(req)
const name = await uploadsController.getUniqueRandomName(length, extension, req.app.get('uploads-set'))
const destination = path.join(uploadsDir, name)
fs.writeFile(destination, file, async error => {
if (error) { return erred(error) }
const data = {
filename: name,
originalname: original,
mimetype: headers.get('content-type').split(';')[0] || '',
size,
albumid
}
infoMap.push({
path: destination,
data
})
iteration++
if (iteration === urls.length) {
if (config.uploads.scan && config.uploads.scan.enabled) {
const scan = await uploadsController.scanFiles(req, infoMap)
if (scan) { return erred(scan) }
}
const result = await uploadsController.formatInfoMap(req, res, user, infoMap)
.catch(erred)
if (!result) { return }
uploadsController.processFilesForDisplay(req, res, result.files, result.existingFiles)
}
})
} catch (error) { erred(error) }
}
}
uploadsController.finishChunks = async (req, res, next) => {
if (!chunkedUploads) {
return res.json({ success: false, description: 'Chunked upload is disabled at the moment.' })
}
let user
if (config.private === true) {
user = await utils.authorize(req, res)
if (!user) { return }
} else if (req.headers.token) {
user = await db.table('users').where('token', req.headers.token).first()
}
if (user && (user.enabled === false || user.enabled === 0)) {
return res.json({ success: false, description: 'This account has been disabled.' })
}
if (user && user.fileLength && !req.headers.filelength) {
req.headers.filelength = user.fileLength
}
let albumid = parseInt(req.headers.albumid || req.params.albumid)
if (isNaN(albumid)) { albumid = null }
return uploadsController.actuallyFinishChunks(req, res, user, albumid)
}
uploadsController.actuallyFinishChunks = async (req, res, user, albumid) => {
const erred = error => {
const isError = error instanceof Error
if (isError) { console.error(error) }
res.status(400).json({
success: false,
description: isError ? error.toString() : error
})
}
const files = req.body.files
if (!files || !(files instanceof Array) || !files.length) { return erred('Invalid "files" property (Array).') }
let iteration = 0
const infoMap = []
for (const file of files) {
if (!file.uuid || typeof file.uuid !== 'string') { return erred('Invalid "uuid" property (string).') }
if (typeof file.count !== 'number' || file.count < 1) { return erred('Invalid "count" property (number).') }
const uuidDir = path.join(chunksDir, file.uuid)
fs.readdir(uuidDir, async (error, chunkNames) => {
if (error) {
if (error.code === 'ENOENT') { return erred('UUID is not being used.') }
return erred(error)
}
if (file.count < chunkNames.length) { return erred('Chunks count mismatch.') }
const extension = typeof file.original === 'string' ? utils.extname(file.original) : ''
if (uploadsController.isExtensionFiltered(extension)) {
return erred(`${extension.substr(1).toUpperCase()} files are not permitted due to security reasons.`)
}
const length = uploadsController.getFileNameLength(req)
const name = await uploadsController.getUniqueRandomName(length, extension, req.app.get('uploads-set'))
.catch(erred)
if (!name) { return }
const destination = path.join(uploadsDir, name)
// Sort chunk names
chunkNames.sort()
// Get total chunks size
const chunksTotalSize = await uploadsController.getTotalSize(uuidDir, chunkNames)
.catch(erred)
if (typeof chunksTotalSize !== 'number') { return }
if (chunksTotalSize > maxSizeBytes) {
// Delete all chunks and remove chunks dir
const chunksCleaned = await uploadsController.cleanUpChunks(uuidDir, chunkNames)
.catch(erred)
if (!chunksCleaned) { return }
return erred(`Total chunks size is bigger than ${maxSize}.`)
}
// Append all chunks
const destFileStream = fs.createWriteStream(destination, { flags: 'a' })
const chunksAppended = await uploadsController.appendToStream(destFileStream, uuidDir, chunkNames)
.catch(erred)
if (!chunksAppended) { return }
// Delete all chunks and remove chunks dir
const chunksCleaned = await uploadsController.cleanUpChunks(uuidDir, chunkNames)
.catch(erred)
if (!chunksCleaned) { return }
const data = {
filename: name,
originalname: file.original || '',
mimetype: file.type || '',
size: file.size || 0
}
data.albumid = parseInt(file.albumid)
if (isNaN(data.albumid)) { data.albumid = albumid }
infoMap.push({
path: destination,
data
})
iteration++
if (iteration === files.length) {
if (config.uploads.scan && config.uploads.scan.enabled) {
const scan = await uploadsController.scanFiles(req, infoMap)
if (scan) { return erred(scan) }
}
const result = await uploadsController.formatInfoMap(req, res, user, infoMap)
.catch(erred)
if (!result) { return }
uploadsController.processFilesForDisplay(req, res, result.files, result.existingFiles)
}
})
}
}
uploadsController.getTotalSize = (uuidDir, chunkNames) => {
return new Promise((resolve, reject) => {
let size = 0
const stat = i => {
if (i === chunkNames.length) { return resolve(size) }
fs.stat(path.join(uuidDir, chunkNames[i]), (error, stats) => {
if (error) { return reject(error) }
size += stats.size
stat(i + 1)
})
}
stat(0)
})
}
uploadsController.appendToStream = (destFileStream, uuidDr, chunkNames) => {
return new Promise((resolve, reject) => {
const append = i => {
if (i === chunkNames.length) {
destFileStream.end()
return resolve(true)
}
fs.createReadStream(path.join(uuidDr, chunkNames[i]))
.on('end', () => {
append(i + 1)
})
.on('error', error => {
console.error(error)
destFileStream.end()
return reject(error)
})
.pipe(destFileStream, { end: false })
}
append(0)
})
}
uploadsController.cleanUpChunks = (uuidDir, chunkNames) => {
return new Promise(async (resolve, reject) => {
await Promise.all(chunkNames.map(chunkName => {
return new Promise((resolve, reject) => {
const chunkPath = path.join(uuidDir, chunkName)
fs.unlink(chunkPath, error => {
if (error && error.code !== 'ENOENT') {
console.error(error)
return reject(error)
}
resolve()
})
})
})).catch(reject)
fs.rmdir(uuidDir, error => {
if (error) { return reject(error) }
resolve(true)
})
})
}
uploadsController.formatInfoMap = (req, res, user, infoMap) => {
return new Promise(async resolve => {
let iteration = 0
const files = []
const existingFiles = []
const albumsAuthorized = {}
for (const info of infoMap) {
// Check if the file exists by checking hash and size
const hash = crypto.createHash('md5')
const stream = fs.createReadStream(info.path)
stream.on('data', data => {
hash.update(data, 'utf8')
})
stream.on('end', async () => {
const fileHash = hash.digest('hex')
const dbFile = await db.table('files')
.where(function () {
if (user === undefined) {
this.whereNull('userid')
} else {
this.where('userid', user.id)
}
})
.where({
hash: fileHash,
size: info.data.size
})
.first()
if (!dbFile) {
if (info.data.albumid && albumsAuthorized[info.data.albumid] === undefined) {
const authorized = await db.table('albums')
.where({
id: info.data.albumid,
userid: user.id
})
.first()
albumsAuthorized[info.data.albumid] = Boolean(authorized)
}
files.push({
name: info.data.filename,
original: info.data.originalname,
type: info.data.mimetype,
size: info.data.size,
hash: fileHash,
ip: req.ip,
albumid: albumsAuthorized[info.data.albumid] ? info.data.albumid : null,
userid: user !== undefined ? user.id : null,
timestamp: Math.floor(Date.now() / 1000)
})
} else {
utils.deleteFile(info.data.filename).catch(console.error)
const set = req.app.get('uploads-set')
if (set) {
const identifier = info.data.filename.split('.')[0]
set.delete(identifier)
// console.log(`Removed ${identifier} from identifiers cache (formatInfoMap)`)
}
existingFiles.push(dbFile)
}
iteration++
if (iteration === infoMap.length) {
resolve({ files, existingFiles })
}
})
}
})
}
uploadsController.scanFiles = (req, infoMap) => {
return new Promise(async (resolve, reject) => {
const scanner = req.app.get('clam-scanner')
let iteration = 0
for (const info of infoMap) {
scanner.scanFile(info.path).then(reply => {
if (!reply.includes('OK') || reply.includes('FOUND')) {
// eslint-disable-next-line no-control-regex
const virus = reply.replace(/^stream: /, '').replace(/ FOUND\u0000$/, '')
console.log(`ClamAV: ${info.data.filename}: ${virus} FOUND.`)
return resolve(virus)
}
iteration++
if (iteration === infoMap.length) {
resolve(null)
}
}).catch(reject)
}
}).then(virus => {
if (!virus) { return false }
// If there is at least one dirty file, then delete all files
const set = req.app.get('uploads-set')
infoMap.forEach(info => {
utils.deleteFile(info.data.filename).catch(console.error)
if (set) {
const identifier = info.data.filename.split('.')[0]
set.delete(identifier)
// console.log(`Removed ${identifier} from identifiers cache (formatInfoMap)`)
}
})
// Unfortunately, we will only be returning name of the first virus
// even if the current session was made up by multiple virus types
return `Virus detected: ${virus}.`
}).catch(error => {
console.error(`ClamAV: ${error.toString()}.`)
return `ClamAV: ${error.code}, please contact site owner.`
})
}
uploadsController.processFilesForDisplay = async (req, res, files, existingFiles) => {
const responseFiles = []
if (files.length) {
// Insert new files to DB
await db.table('files').insert(files)
for (const file of files) {
responseFiles.push(file)
}
}
if (existingFiles.length) {
for (const file of existingFiles) {
responseFiles.push(file)
}
}
// We send response first before generating thumbnails and updating album timestamps
res.json({
success: true,
files: responseFiles.map(file => {
return {
name: file.name,
size: file.size,
url: `${config.domain}/${file.name}`
}
})
})
const albumids = []
for (const file of files) {
if (file.albumid && !albumids.includes(file.albumid)) {
albumids.push(file.albumid)
}
if (utils.mayGenerateThumb(utils.extname(file.name))) {
utils.generateThumbs(file.name)
}
}
if (albumids.length) {
await db.table('albums')
.whereIn('id', albumids)
.update('editedAt', Math.floor(Date.now() / 1000))
.catch(console.error)
}
}
uploadsController.delete = async (req, res) => {
const id = parseInt(req.body.id)
req.body.field = 'id'
req.body.values = isNaN(id) ? undefined : [id]
delete req.body.id
return uploadsController.bulkDelete(req, res)
}
uploadsController.bulkDelete = async (req, res) => {
const user = await utils.authorize(req, res)
if (!user) { return }
const field = req.body.field || 'id'
const values = req.body.values
if (!values || values.constructor.name !== 'Array' || !values.length) {
return res.json({ success: false, description: 'No array of files specified.' })
}
const failed = await utils.bulkDeleteFiles(field, values, user, req.app.get('uploads-set'))
if (failed.length < values.length) {
return res.json({ success: true, failed })
}
return res.json({ success: false, description: 'Could not delete any files.' })
}
uploadsController.list = async (req, res) => {
const user = await utils.authorize(req, res)
if (!user) { return }
let offset = req.params.page
if (offset === undefined) { offset = 0 }
// Headers is string-only, this seem to be the safest and lightest
const all = req.headers.all === '1'
const ismoderator = perms.is(user, 'moderator')
if (all && !ismoderator) { return res.json(403) }
const files = await db.table('files')
.where(function () {
if (req.params.id === undefined) {
this.where('id', '<>', '')
} else {
this.where('albumid', req.params.id)
}
})
.where(function () {
if (!all || !ismoderator) {
this.where('userid', user.id)
}
})
.orderBy('id', 'DESC')
.limit(25)
.offset(25 * offset)
.select('id', 'albumid', 'timestamp', 'name', 'userid', 'size')
const albums = await db.table('albums')
.where(function () {
this.where('enabled', 1)
if (!all || !ismoderator) {
this.where('userid', user.id)
}
})
const basedomain = config.domain
const userids = []
for (const file of files) {
file.file = `${basedomain}/${file.name}`
file.album = ''
if (file.albumid !== undefined) {
for (const album of albums) {
if (file.albumid === album.id) {
file.album = album.name
}
}
}
// Only push usernames if we are a moderator
if (all && ismoderator) {
if (file.userid !== undefined && file.userid !== null && file.userid !== '') {
userids.push(file.userid)
}
}
file.extname = utils.extname(file.name)
if (utils.mayGenerateThumb(file.extname)) {
file.thumb = `${basedomain}/thumbs/${file.name.slice(0, -file.extname.length)}.png`
}
}
// If we are a normal user, send response
if (!ismoderator) { return res.json({ success: true, files }) }
// If we are a moderator but there are no uploads attached to a user, send response
if (userids.length === 0) { return res.json({ success: true, files }) }
const users = await db.table('users').whereIn('id', userids)
for (const dbUser of users) {
for (const file of files) {
if (file.userid === dbUser.id) {
file.username = dbUser.username
}
}
}
return res.json({ success: true, files })
}
module.exports = uploadsController