filesafe/controllers/uploadController.js

const blake3 = require('blake3')
const fetch = require('node-fetch')
const fs = require('fs')
const multer = require('multer')
const path = require('path')
const randomstring = require('randomstring')
const searchQuery = require('search-query-parser')
const paths = require('./pathsController')
const perms = require('./permissionController')
const utils = require('./utilsController')
const apiErrorsHandler = require('./handlers/apiErrorsHandler.js')
const ClientError = require('./utils/ClientError')
const multerStorage = require('./utils/multerStorage')
const ServerError = require('./utils/ServerError')
const config = require('./../config')
const logger = require('./../logger')
const db = require('knex')(config.database)

const self = {
  onHold: new Set()
}

/** Preferences */

const fileIdentifierLengthFallback = 32
const fileIdentifierLengthChangeable = !config.uploads.fileIdentifierLength.force &&
  typeof config.uploads.fileIdentifierLength.min === 'number' &&
  typeof config.uploads.fileIdentifierLength.max === 'number'

const maxSize = parseInt(config.uploads.maxSize)
const maxSizeBytes = maxSize * 1e6
const urlMaxSizeBytes = parseInt(config.uploads.urlMaxSize) * 1e6

const maxFilesPerUpload = 20

const chunkedUploads = config.uploads.chunkSize &&
  typeof config.uploads.chunkSize === 'object' &&
  config.uploads.chunkSize.default
const chunkedUploadsTimeout = config.uploads.chunkSize.timeout || 1800000
const chunksData = {}
// Hard-coded min chunk size of 1 MB (e.g. 50 MB = max 50 chunks)
const maxChunksCount = maxSize
// Use fs.copyFile() instead of fs.rename() if chunks dir is NOT inside uploads dir
const chunksCopyFile = !paths.chunks.startsWith(paths.uploads)

const extensionsFilter = Array.isArray(config.extensionsFilter) &&
  config.extensionsFilter.length
const urlExtensionsFilter = Array.isArray(config.uploads.urlExtensionsFilter) &&
  config.uploads.urlExtensionsFilter.length
const temporaryUploads = Array.isArray(config.uploads.temporaryUploadAges) &&
  config.uploads.temporaryUploadAges.length

/** Chunks helper class & function **/

class ChunksData {
  constructor (uuid, root) {
    this.uuid = uuid
    this.root = root
    this.filename = 'tmp'
    this.chunks = 0
    this.stream = null
    this.hasher = null
  }

  onTimeout () {
    if (this.stream && !this.stream.writableEnded) {
      this.stream.end()
    }
    if (this.hasher) {
      this.hasher.dispose()
    }
    self.cleanUpChunks(this.uuid, true)
  }

  setTimeout (delay) {
    this.clearTimeout()
    this._timeout = setTimeout(this.onTimeout.bind(this), delay)
  }

  clearTimeout () {
    if (this._timeout) {
      clearTimeout(this._timeout)
    }
  }
}

const initChunks = async uuid => {
  if (chunksData[uuid] === undefined) {
    const root = path.join(paths.chunks, uuid)
    try {
      await paths.access(root)
    } catch (err) {
      // Re-throw error
      if (err && err.code !== 'ENOENT') throw err
      await paths.mkdir(root)
    }
    chunksData[uuid] = new ChunksData(uuid, root)
  }
  chunksData[uuid].setTimeout(chunkedUploadsTimeout)
  return chunksData[uuid]
}

/** Multer */

const executeMulter = multer({
  // Guide: https://github.com/expressjs/multer#limits
  limits: {
    fileSize: maxSizeBytes,
    // Maximum number of non-file fields.
    // Dropzone.js will add 6 extra fields for chunked uploads.
    // We don't use them for anything else.
    fields: 6,
    // Maximum number of file fields.
    // Chunked uploads still need to provide ONLY 1 file field.
    // Otherwise, only one of the files will end up being properly stored,
    // and that will also be as a chunk.
    files: maxFilesPerUpload
  },
  fileFilter (req, file, cb) {
    file.extname = utils.extname(file.originalname)
    if (self.isExtensionFiltered(file.extname)) {
      return cb(new ClientError(`${file.extname ? `${file.extname.substr(1).toUpperCase()} files` : 'Files with no extension'} are not permitted.`))
    }

    // Re-map Dropzone keys so people can manually use the API without prepending 'dz'
    for (const key in req.body) {
      if (!/^dz/.test(key)) continue
      req.body[key.replace(/^dz/, '')] = req.body[key]
      delete req.body[key]
    }

    if (req.body.chunkindex !== undefined && !chunkedUploads) {
      return cb(new ClientError('Chunked uploads are disabled at the moment.'))
    } else {
      return cb(null, true)
    }
  },
  storage: multerStorage({
    destination (req, file, cb) {
      // Is file a chunk!?
      file._isChunk = chunkedUploads && req.body.uuid !== undefined && req.body.chunkindex !== undefined

      if (file._isChunk) {
        initChunks(req.body.uuid)
          .then(chunksData => {
            file._chunksData = chunksData
            cb(null, chunksData.root)
          })
          .catch(error => {
            logger.error(error)
            return cb(new ServerError('Could not process the chunked upload. Try again?'))
          })
      } else {
        return cb(null, paths.uploads)
      }
    },

    filename (req, file, cb) {
      if (file._isChunk) {
        return cb(null, chunksData[req.body.uuid].filename)
      } else {
        const length = self.parseFileIdentifierLength(req.headers.filelength)
        return self.getUniqueRandomName(length, file.extname)
          .then(name => cb(null, name))
          .catch(error => cb(error))
      }
    },

    clamscan: utils.clamscan
  })
}).array('files[]')

/** Helper functions */

self.isExtensionFiltered = extname => {
  // If empty extension needs to be filtered
  if (!extname && config.filterNoExtension) return true

  // If there are extensions that have to be filtered
  if (extname && extensionsFilter) {
    const match = config.extensionsFilter.includes(extname.toLowerCase())
    const whitelist = config.extensionsFilterMode === 'whitelist'
    if ((!whitelist && match) || (whitelist && !match)) return true
  }

  return false
}

self.parseFileIdentifierLength = fileLength => {
  if (!config.uploads.fileIdentifierLength) return fileIdentifierLengthFallback

  const parsed = parseInt(fileLength)
  if (isNaN(parsed) ||
    !fileIdentifierLengthChangeable ||
    parsed < config.uploads.fileIdentifierLength.min ||
    parsed > config.uploads.fileIdentifierLength.max) {
    return config.uploads.fileIdentifierLength.default || fileIdentifierLengthFallback
  } else {
    return parsed
  }
}

self.getUniqueRandomName = async (length, extension) => {
  for (let i = 0; i < utils.idMaxTries; i++) {
    const identifier = randomstring.generate(length)
    const name = identifier + extension
    if (config.uploads.cacheFileIdentifiers) {
      if (utils.idSet.has(identifier)) {
        logger.log(`Identifier ${identifier} is already in use (${i + 1}/${utils.idMaxTries}).`)
        continue
      }
      utils.idSet.add(identifier)
      logger.debug(`Added ${identifier} to identifiers cache`)
    } else if (config.uploads.queryDbForFileCollisions) {
      if (self.onHold.has(identifier)) continue

      // Put token on-hold (wait for it to be inserted to DB)
      self.onHold.add(identifier)

      const file = await db.table('files')
        .whereRaw('?? like ?', ['name', `${identifier}.%`])
        .select('id')
        .first()
      if (file) {
        self.onHold.delete(identifier)
        logger.log(`Identifier ${identifier} is already in use (${i + 1}/${utils.idMaxTries}).`)
        continue
      }
    } else {
      try {
        await paths.access(path.join(paths.uploads, name))
        logger.log(`${name} is already in use (${i + 1}/${utils.idMaxTries}).`)
        continue
      } catch (error) {
        // Re-throw non-ENOENT error
        if (error & error.code !== 'ENOENT') throw error
      }
    }
    return name
  }

  throw new ServerError('Failed to allocate a unique name for the upload. Try again?')
}

self.parseUploadAge = age => {
  if (age === undefined || age === null) {
    return config.uploads.temporaryUploadAges[0]
  }

  const parsed = parseFloat(age)
  if (config.uploads.temporaryUploadAges.includes(parsed)) {
    return parsed
  } else {
    return null
  }
}

self.parseStripTags = stripTags => {
  if (!config.uploads.stripTags) return false

  if (config.uploads.stripTags.force || stripTags === undefined) {
    return config.uploads.stripTags.default
  }

  return Boolean(parseInt(stripTags))
}

/** Uploads */

self.upload = async (req, res, next) => {
  try {
    let user
    if (config.private === true) {
      user = await utils.authorize(req)
    } else if (req.headers.token) {
      user = await utils.assertUser(req.headers.token)
    }

    if (config.privateUploadGroup) {
      if (!user || !perms.is(user, config.privateUploadGroup)) {
        throw new ClientError(config.privateUploadCustomResponse || 'Your usergroup is not permitted to upload new files.', { statusCode: 403 })
      }
    }

    let albumid = parseInt(req.headers.albumid || req.params.albumid)
    if (isNaN(albumid)) albumid = null

    let age = null
    if (temporaryUploads) {
      age = self.parseUploadAge(req.headers.age)
      if (!age && !config.uploads.temporaryUploadAges.includes(0)) {
        throw new ClientError('Permanent uploads are not permitted.', { statusCode: 403 })
      }
    }

    const func = req.body.urls ? self.actuallyUploadUrls : self.actuallyUploadFiles
    await func(req, res, user, albumid, age)
  } catch (error) {
    return apiErrorsHandler(error, req, res, next)
  }
}

self.actuallyUploadFiles = async (req, res, user, albumid, age) => {
  const error = await new Promise(resolve => {
    return executeMulter(req, res, err => resolve(err))
  })

  if (error) {
    const suppress = [
      'LIMIT_FILE_SIZE',
      'LIMIT_UNEXPECTED_FILE'
    ]
    if (suppress.includes(error.code)) {
      throw new ClientError(error.toString())
    } else {
      throw error
    }
  }

  if (!req.files || !req.files.length) {
    throw new ClientError('No files.')
  }

  // If chunked uploads is enabled and the uploaded file is a chunk, then just say that it was a success
  const uuid = req.body.uuid
  if (chunkedUploads && chunksData[uuid] !== undefined) {
    req.files.forEach(file => {
      chunksData[uuid].chunks++
    })
    return res.json({ success: true })
  }

  const infoMap = req.files.map(file => {
    file.albumid = albumid
    file.age = age
    return {
      path: path.join(paths.uploads, file.filename),
      data: file
    }
  })

  if (config.filterEmptyFile && infoMap.some(file => file.data.size === 0)) {
    // Unlink all files when at least one file is an empty file
    // Should continue even when encountering errors
    await Promise.all(infoMap.map(info =>
      utils.unlinkFile(info.data.filename).catch(logger.error)
    ))

    throw new ClientError('Empty files are not allowed.')
  }

  if (utils.clamscan.instance) {
    let scanResult
    if (utils.clamscan.passthrough) {
      scanResult = await self.assertPassthroughScans(req, user, infoMap)
    } else {
      scanResult = await self.scanFiles(req, user, infoMap)
    }
    if (scanResult) {
      throw new ClientError(scanResult)
    }
  }

  await self.stripTags(req, infoMap)

  const result = await self.storeFilesToDb(req, res, user, infoMap)
  return self.sendUploadResponse(req, res, user, result)
}

/** URL uploads */

self.actuallyUploadUrls = async (req, res, user, albumid, age) => {
  if (!config.uploads.urlMaxSize) {
    throw new ClientError('Upload by URLs is disabled at the moment.', { statusCode: 403 })
  }

  const urls = req.body.urls
  if (!urls || !(urls instanceof Array)) {
    throw new ClientError('Missing "urls" property (array).')
  }

  if (urls.length > maxFilesPerUpload) {
    throw new ClientError(`Maximum ${maxFilesPerUpload} URLs at a time.`)
  }

  const downloaded = []
  const infoMap = []
  try {
    await Promise.all(urls.map(async url => {
      const original = path.basename(url).split(/[?#]/)[0]
      const extname = utils.extname(original)

      // Extensions filter
      let filtered = false
      if (urlExtensionsFilter && ['blacklist', 'whitelist'].includes(config.uploads.urlExtensionsFilterMode)) {
        const match = config.uploads.urlExtensionsFilter.includes(extname.toLowerCase())
        const whitelist = config.uploads.urlExtensionsFilterMode === 'whitelist'
        filtered = ((!whitelist && match) || (whitelist && !match))
      } else {
        filtered = self.isExtensionFiltered(extname)
      }

      if (filtered) {
        throw new ClientError(`${extname ? `${extname.substr(1).toUpperCase()} files` : 'Files with no extension'} are not permitted.`)
      }

      if (config.uploads.urlProxy) {
        url = config.uploads.urlProxy
          .replace(/{url}/g, encodeURIComponent(url))
          .replace(/{url-noprot}/g, encodeURIComponent(url.replace(/^https?:\/\//, '')))
      }

      const length = self.parseFileIdentifierLength(req.headers.filelength)
      const name = await self.getUniqueRandomName(length, extname)

      const destination = path.join(paths.uploads, name)
      const outStream = fs.createWriteStream(destination)
      const hash = blake3.createHash()

      // Push to array early, so regardless of its progress it will be deleted on errors
      downloaded.push(destination)

      // Limit max response body size with maximum allowed size
      const fetchFile = await fetch(url, { size: urlMaxSizeBytes })
        .then(res => new Promise((resolve, reject) => {
          if (res.status === 200) {
            const onerror = error => {
              hash.dispose()
              reject(error)
            }
            outStream.on('error', onerror)
            res.body.on('error', onerror)
            res.body.on('data', d => hash.update(d))

            res.body.pipe(outStream)
            outStream.on('finish', () => resolve(res))
          } else {
            resolve(res)
          }
        }))

      if (fetchFile.status !== 200) {
        throw new ServerError(`${fetchFile.status} ${fetchFile.statusText}`)
      }

      infoMap.push({
        path: destination,
        data: {
          filename: name,
          originalname: original,
          extname,
          mimetype: fetchFile.headers.get('content-type').split(';')[0] || '',
          size: outStream.bytesWritten,
          hash: hash.digest('hex'),
          albumid,
          age
        }
      })
    }))

    // If no errors encountered, clear cache of downloaded files
    downloaded.length = 0

    if (utils.clamscan.instance) {
      const scanResult = await self.scanFiles(req, user, infoMap)
      if (scanResult) throw new ClientError(scanResult)
    }

    const result = await self.storeFilesToDb(req, res, user, infoMap)
    await self.sendUploadResponse(req, res, user, result)
  } catch (error) {
    // Unlink all downloaded files when at least one file threw an error from the for-loop
    // Should continue even when encountering errors
    if (downloaded.length) {
      await Promise.all(downloaded.map(file =>
        utils.unlinkFile(file).catch(logger.error)
      ))
    }

    const errorString = error.toString()
    const suppress = [
      / over limit:/
    ]
    if (suppress.some(t => t.test(errorString))) {
      throw new ClientError(errorString)
    } else {
      throw error
    }
  }
}

/** Chunk uploads */

self.finishChunks = async (req, res, next) => {
  try {
    if (!chunkedUploads) {
      throw new ClientError('Chunked upload is disabled.', { statusCode: 403 })
    }

    let user
    if (config.private === true) {
      user = await utils.authorize(req)
      if (!user) return
    } else if (req.headers.token) {
      user = await utils.assertUser(req.headers.token)
    }

    await self.actuallyFinishChunks(req, res, user)
  } catch (error) {
    return apiErrorsHandler(error, req, res, next)
  }
}

self.actuallyFinishChunks = async (req, res, user) => {
  const check = file => typeof file.uuid !== 'string' ||
    !chunksData[file.uuid] ||
    chunksData[file.uuid].chunks < 2

  const files = req.body.files
  if (!Array.isArray(files) || !files.length || files.some(check)) {
    throw new ClientError('Bad request.')
  }

  const infoMap = []
  try {
    await Promise.all(files.map(async file => {
      // Close stream
      chunksData[file.uuid].stream.end()

      if (chunksData[file.uuid].chunks > maxChunksCount) {
        throw new ClientError('Too many chunks.')
      }

      file.extname = typeof file.original === 'string' ? utils.extname(file.original) : ''
      if (self.isExtensionFiltered(file.extname)) {
        throw new ClientError(`${file.extname ? `${file.extname.substr(1).toUpperCase()} files` : 'Files with no extension'} are not permitted.`)
      }

      if (temporaryUploads) {
        file.age = self.parseUploadAge(file.age)
        if (!file.age && !config.uploads.temporaryUploadAges.includes(0)) {
          throw new ClientError('Permanent uploads are not permitted.')
        }
      }

      file.size = chunksData[file.uuid].stream.bytesWritten
      if (config.filterEmptyFile && file.size === 0) {
        throw new ClientError('Empty files are not allowed.')
      } else if (file.size > maxSizeBytes) {
        throw new ClientError(`File too large. Chunks are bigger than ${maxSize} MB.`)
      }

      // Double-check file size
      const tmpfile = path.join(chunksData[file.uuid].root, chunksData[file.uuid].filename)
      const lstat = await paths.lstat(tmpfile)
      if (lstat.size !== file.size) {
        throw new ClientError(`File size mismatched (${lstat.size} vs. ${file.size}).`)
      }

      // Generate name
      const length = self.parseFileIdentifierLength(file.filelength)
      const name = await self.getUniqueRandomName(length, file.extname)

      // Move tmp file to final destination
      // For fs.copyFile(), tmpfile will eventually be unlinked by self.cleanUpChunks()
      const destination = path.join(paths.uploads, name)
      if (chunksCopyFile) {
        await paths.copyFile(tmpfile, destination)
      } else {
        await paths.rename(tmpfile, destination)
      }
      const hash = chunksData[file.uuid].hasher.digest('hex')

      // Continue even when encountering errors
      await self.cleanUpChunks(file.uuid).catch(logger.error)

      let albumid = parseInt(file.albumid)
      if (isNaN(albumid)) albumid = null

      const data = {
        filename: name,
        originalname: file.original || '',
        extname: file.extname,
        mimetype: file.type || '',
        size: file.size,
        hash,
        albumid,
        age: file.age
      }

      infoMap.push({ path: destination, data })
    }))

    if (utils.clamscan.instance) {
      const scanResult = await self.scanFiles(req, user, infoMap)
      if (scanResult) throw new ClientError(scanResult)
    }

    await self.stripTags(req, infoMap)

    const result = await self.storeFilesToDb(req, res, user, infoMap)
    await self.sendUploadResponse(req, res, user, result)
  } catch (error) {
    // Dispose unfinished hasher and clean up leftover chunks
    // Should continue even when encountering errors
    files.forEach(file => {
      if (chunksData[file.uuid] === undefined) return
      try {
        if (chunksData[file.uuid].hasher) {
          chunksData[file.uuid].hasher.dispose()
        }
      } catch (_) {}
      self.cleanUpChunks(file.uuid).catch(logger.error)
    })

    // Re-throw error
    throw error
  }
}

self.cleanUpChunks = async (uuid, onTimeout) => {
  // Remove tmp file
  await paths.unlink(path.join(chunksData[uuid].root, chunksData[uuid].filename))
    .catch(error => {
      // Re-throw non-ENOENT error
      if (error.code !== 'ENOENT') logger.error(error)
    })

  // Remove UUID dir
  await paths.rmdir(chunksData[uuid].root)

  // Delete cached chunks data
  if (!onTimeout) chunksData[uuid].clearTimeout()
  delete chunksData[uuid]
}

/** Virus scanning (ClamAV) */

self.assertPassthroughScans = async (req, user, infoMap) => {
  const foundThreats = []
  const unableToScan = []

  for (const info of infoMap) {
    if (info.data.clamscan) {
      if (info.data.clamscan.isInfected) {
        logger.log(`[ClamAV]: ${info.data.filename}: ${info.data.clamscan.viruses.join(', ')}`)
        foundThreats.push(...info.data.clamscan.viruses)
      } else if (info.data.clamscan.isInfected === null) {
        logger.log(`[ClamAV]: ${info.data.filename}: Unable to scan`)
        unableToScan.push(info.data.filename)
      }
    } else {
      unableToScan.push(info.data.filename)
    }
  }

  let result = ''
  if (foundThreats.length) {
    const more = foundThreats.length > 1
    result = `Threat${more ? 's' : ''} detected: ${foundThreats[0]}${more ? ', and more' : ''}.`
  } else if (unableToScan.length) {
    const more = unableToScan.length > 1
    result = `Unable to scan: ${unableToScan[0]}${more ? ', and more' : ''}.`
  }

  if (result) {
    // Unlink all files when at least one threat is found
    // Should continue even when encountering errors
    await Promise.all(infoMap.map(info =>
      utils.unlinkFile(info.data.filename).catch(logger.error)
    ))
  }

  return result
}

self.scanFiles = async (req, user, infoMap) => {
  if (user && utils.clamscan.groupBypass && perms.is(user, utils.clamscan.groupBypass)) {
    logger.debug(`[ClamAV]: Skipping ${infoMap.length} file(s), ${utils.clamscan.groupBypass} group bypass`)
    return false
  }

  const foundThreats = []
  const unableToScan = []
  const result = await Promise.all(infoMap.map(async info => {
    if (utils.clamscan.whitelistExtensions && utils.clamscan.whitelistExtensions.includes(info.data.extname)) {
      logger.debug(`[ClamAV]: Skipping ${info.data.filename}, extension whitelisted`)
      return
    }

    if (utils.clamscan.maxSize && info.data.size > utils.clamscan.maxSize) {
      logger.debug(`[ClamAV]: Skipping ${info.data.filename}, size ${info.data.size} > ${utils.clamscan.maxSize}`)
      return
    }

    logger.debug(`[ClamAV]: ${info.data.filename}: Scanning\u2026`)
    const response = await utils.clamscan.instance.isInfected(info.path)
    if (response.isInfected) {
      logger.log(`[ClamAV]: ${info.data.filename}: ${response.viruses.join(', ')}`)
      foundThreats.push(...response.viruses)
    } else if (response.isInfected === null) {
      logger.log(`[ClamAV]: ${info.data.filename}: Unable to scan`)
      unableToScan.push(info.data.filename)
    }
  })).then(() => {
    if (foundThreats.length) {
      const more = foundThreats.length > 1
      return `Threat${more ? 's' : ''} detected: ${foundThreats[0]}${more ? ', and more' : ''}.`
    } else if (unableToScan.length) {
      const more = unableToScan.length > 1
      return `Unable to scan: ${unableToScan[0]}${more ? ', and more' : ''}.`
    }
  }).catch(error => {
    logger.error(`[ClamAV]: ${infoMap.map(info => info.data.filename).join(', ')}: ${error.toString()}`)
    return 'An unexpected error occurred with ClamAV, please contact the site owner.'
  })

  if (result) {
    // Unlink all files when at least one threat is found OR any errors occurred
    // Should continue even when encountering errors
    await Promise.all(infoMap.map(info =>
      utils.unlinkFile(info.data.filename).catch(logger.error)
    ))
  }

  return result
}

/** Strip tags (EXIF, etc.) */

self.stripTags = async (req, infoMap) => {
  if (!self.parseStripTags(req.headers.striptags)) return

  try {
    await Promise.all(infoMap.map(info =>
      utils.stripTags(info.data.filename, info.data.extname)
    ))
  } catch (error) {
    // Unlink all files when at least one threat is found OR any errors occurred
    // Should continue even when encountering errors
    await Promise.all(infoMap.map(info =>
      utils.unlinkFile(info.data.filename).catch(logger.error)
    ))

    // Re-throw error
    throw error
  }
}

/** Database functions */

self.storeFilesToDb = async (req, res, user, infoMap) => {
  const files = []
  const exists = []
  const albumids = []

  await Promise.all(infoMap.map(async info => {
    // Check if the file exists by checking its hash and size
    const dbFile = await db.table('files')
      .where(function () {
        if (user === undefined) {
          this.whereNull('userid')
        } else {
          this.where('userid', user.id)
        }
      })
      .where({
        hash: info.data.hash,
        size: info.data.size
      })
      // Select expirydate to display expiration date of existing files as well
      .select('name', 'expirydate')
      .first()

    if (dbFile) {
      // Continue even when encountering errors
      await utils.unlinkFile(info.data.filename).catch(logger.error)
      logger.debug(`Unlinked ${info.data.filename} since a duplicate named ${dbFile.name} exists`)

      // If on /nojs route, append original file name reported by client
      if (req.path === '/nojs') {
        dbFile.original = info.data.originalname
      }

      exists.push(dbFile)
      return
    }

    const timestamp = Math.floor(Date.now() / 1000)
    const data = {
      name: info.data.filename,
      original: info.data.originalname,
      type: info.data.mimetype,
      size: info.data.size,
      hash: info.data.hash,
      // Only disable if explicitly set to false in config
      ip: config.uploads.storeIP !== false ? req.ip : null,
      timestamp
    }

    if (user) {
      data.userid = user.id
      data.albumid = info.data.albumid
      if (data.albumid !== null && !albumids.includes(data.albumid)) {
        albumids.push(data.albumid)
      }
    }

    if (info.data.age) {
      data.expirydate = data.timestamp + (info.data.age * 3600) // Hours to seconds
    }

    files.push(data)

    // Generate thumbs, but do not wait
    if (utils.mayGenerateThumb(info.data.extname)) {
      utils.generateThumbs(info.data.filename, info.data.extname, true).catch(logger.error)
    }
  }))

  if (files.length) {
    let authorizedIds = []
    if (albumids.length) {
      authorizedIds = await db.table('albums')
        .where({ userid: user.id })
        .whereIn('id', albumids)
        .select('id')
        .then(rows => rows.map(row => row.id))

      // Remove albumid if user do not own the album
      for (const file of files) {
        if (file.albumid !== null && !authorizedIds.includes(file.albumid)) {
          file.albumid = null
        }
      }
    }

    // Insert new files to DB
    await db.table('files').insert(files)
    utils.invalidateStatsCache('uploads')

    if (config.uploads.queryDbForFileCollisions) {
      for (const file of files) {
        const extname = utils.extname(file.name)
        const identifier = file.name.slice(0, -(extname.length))
        self.onHold.delete(identifier)
      }
    }

    // Update albums' timestamp
    if (authorizedIds.length) {
      await db.table('albums')
        .whereIn('id', authorizedIds)
        .update('editedAt', Math.floor(Date.now() / 1000))
      utils.invalidateAlbumsCache(authorizedIds)
    }
  }

  return [...files, ...exists]
}

/** Final response */

self.sendUploadResponse = async (req, res, user, result) => {
  // Send response
  return res.json({
    success: true,
    files: result.map(file => {
      const map = {
        name: file.name,
        url: `${config.domain}/${file.name}`
      }

      // If a temporary upload, add expiry date
      if (file.expirydate) {
        map.expirydate = file.expirydate
      }

      // If on /nojs route, add original name
      if (req.path === '/nojs') {
        map.original = file.original
      }

      // If uploaded by user, add delete URL (intended for ShareX and its derivatives)
      // Homepage uploader will not use this (use dashboard instead)
      // REVISION: I wasn't aware ShareX wouldn't do a basic GET request to this API,
      // which I hoped would then use the token header in the downloadable ShareX config file.
      // At its current state, this isn't really usable.
      /*
      if (user)
        map.deleteUrl = `${config.homeDomain}/api/upload/delete/${file.name}`
      */

      return map
    })
  })
}

/** Delete uploads */

self.delete = async (req, res, next) => {
  // Map /api/delete requests to /api/bulkdelete
  let body
  if (req.method === 'POST') {
    // Original lolisafe API (this fork uses /api/bulkdelete immediately)
    const id = parseInt(req.body.id)
    body = {
      field: 'id',
      values: isNaN(id) ? undefined : [id]
    }
  } /* else if (req.method === 'GET') {
    // ShareX-compatible API (or other clients that require basic GET-based API)
    const name = req.params.name
    body = {
      field: 'name',
      values: name ? [name] : undefined
    }
  } */

  req.body = body
  return self.bulkDelete(req, res, next)
}

self.bulkDelete = async (req, res, next) => {
  try {
    const user = await utils.authorize(req)

    const field = req.body.field || 'id'
    const values = req.body.values

    if (!Array.isArray(values) || !values.length) {
      throw new ClientError('No array of files specified.')
    }

    const failed = await utils.bulkDeleteFromDb(field, values, user)
    await res.json({ success: true, failed })
  } catch (error) {
    return apiErrorsHandler(error, req, res, next)
  }
}

/** List uploads */

self.list = async (req, res, next) => {
  try {
    const user = await utils.authorize(req)

    const all = req.headers.all === '1'
    const filters = req.headers.filters
    const minoffset = Number(req.headers.minoffset) || 0
    const ismoderator = perms.is(user, 'moderator')
    if (all && !ismoderator) return res.status(403).end()

    const basedomain = config.domain

    // Thresholds for regular users
    const MAX_WILDCARDS_IN_KEY = 2
    const MAX_TEXT_QUERIES = 3 // non-keyed keywords
    const MAX_SORT_KEYS = 1
    const MAX_IS_KEYS = 1

    const filterObj = {
      uploaders: [],
      excludeUploaders: [],
      queries: {
        exclude: {}
      },
      typeIs: [
        'image',
        'video',
        'audio'
      ],
      flags: {}
    }

    const sortObj = {
      // Cast columns to specific type if they are stored differently
      casts: {
        size: 'integer'
      },
      // Columns mapping
      maps: {
        date: 'timestamp',
        expiry: 'expirydate',
        originalname: 'original'
      },
      // Columns with which to use SQLite's NULLS LAST option
      nullsLast: [
        'userid',
        'expirydate',
        'ip'
      ],
      parsed: []
    }

    // Parse glob wildcards into SQL wildcards
    function sqlLikeParser (pattern) {
      // Escape SQL operators
      const escaped = pattern
        .replace(/(?<!\\)%/g, '\\%')
        .replace(/(?<!\\)_/g, '\\_')

      // Look for any glob operators
      const match = pattern.match(/(?<!\\)(\*|\?)/g)
      if (match && match.length) {
        return {
          count: match.length,
          // Replace glob operators with their SQL equivalents
          escaped: escaped
            .replace(/(?<!\\)\*/g, '%')
            .replace(/(?<!\\)\?/g, '_')
        }
      } else {
        return {
          count: 0,
          // Assume partial match
          escaped: `%${escaped}%`
        }
      }
    }

    if (filters) {
      const keywords = []

      if (req.params.id === undefined) keywords.push('albumid')

      // Only allow filtering by 'ip' and 'user' keys when listing all uploads
      if (all) keywords.push('ip', 'user')

      const ranges = [
        'date',
        'expiry'
      ]

      keywords.push('is', 'sort', 'orderby')
      filterObj.queries = searchQuery.parse(filters, {
        keywords,
        ranges,
        tokenize: true,
        alwaysArray: true,
        offsets: false
      })

      // Accept orderby as alternative for sort
      if (filterObj.queries.orderby) {
        if (!filterObj.queries.sort) filterObj.queries.sort = []
        filterObj.queries.sort.push(...filterObj.queries.orderby)
        delete filterObj.queries.orderby
      }

      // For some reason, single value won't be in Array even with 'alwaysArray' option
      if (typeof filterObj.queries.exclude.text === 'string') {
        filterObj.queries.exclude.text = [filterObj.queries.exclude.text]
      }

      // Text (non-keyed keywords) queries
      let textQueries = 0
      if (filterObj.queries.text) textQueries += filterObj.queries.text.length
      if (filterObj.queries.exclude.text) textQueries += filterObj.queries.exclude.text.length

      // Regular user threshold check
      if (!ismoderator && textQueries > MAX_TEXT_QUERIES) {
        throw new ClientError(`Users are only allowed to use ${MAX_TEXT_QUERIES} non-keyed keyword${MAX_TEXT_QUERIES === 1 ? '' : 's'} at a time.`)
      }

      if (filterObj.queries.text) {
        for (let i = 0; i < filterObj.queries.text.length; i++) {
          const result = sqlLikeParser(filterObj.queries.text[i])
          if (!ismoderator && result.count > MAX_WILDCARDS_IN_KEY) {
            throw new ClientError(`Users are only allowed to use ${MAX_WILDCARDS_IN_KEY} wildcard${MAX_WILDCARDS_IN_KEY === 1 ? '' : 's'} per key.`)
          }
          filterObj.queries.text[i] = result.escaped
        }
      }

      if (filterObj.queries.exclude.text) {
        for (let i = 0; i < filterObj.queries.exclude.text.length; i++) {
          const result = sqlLikeParser(filterObj.queries.exclude.text[i])
          if (!ismoderator && result.count > MAX_WILDCARDS_IN_KEY) {
            throw new ClientError(`Users are only allowed to use ${MAX_WILDCARDS_IN_KEY} wildcard${MAX_WILDCARDS_IN_KEY === 1 ? '' : 's'} per key.`)
          }
          filterObj.queries.exclude.text[i] = result.escaped
        }
      }

      for (const key of keywords) {
        let queryIndex = -1
        let excludeIndex = -1

        // Make sure keyword arrays only contain unique values
        if (filterObj.queries[key]) {
          filterObj.queries[key] = filterObj.queries[key].filter((v, i, a) => a.indexOf(v) === i)
          queryIndex = filterObj.queries[key].indexOf('-')
        }
        if (filterObj.queries.exclude[key]) {
          filterObj.queries.exclude[key] = filterObj.queries.exclude[key].filter((v, i, a) => a.indexOf(v) === i)
          excludeIndex = filterObj.queries.exclude[key].indexOf('-')
        }

        // Flag to match NULL values
        const inQuery = queryIndex !== -1
        const inExclude = excludeIndex !== -1
        if (inQuery || inExclude) {
          // Prioritize exclude keys when both types found
          filterObj.flags[`${key}Null`] = inExclude ? false : inQuery
          if (inQuery) {
            if (filterObj.queries[key].length === 1) {
              // Delete key to avoid unexpected behavior
              delete filterObj.queries[key]
            } else {
              filterObj.queries[key].splice(queryIndex, 1)
            }
          }
          if (inExclude) {
            if (filterObj.queries.exclude[key].length === 1) {
              // Delete key to avoid unexpected behavior
              delete filterObj.queries.exclude[key]
            } else {
              filterObj.queries.exclude[key].splice(excludeIndex, 1)
            }
          }
        }
      }

      const parseDate = (date, minoffset, resetMs) => {
        // [YYYY][/MM][/DD] [HH][:MM][:SS]
        // e.g. 2020/01/01 00:00:00, 2018/01/01 06, 2019/11, 12:34:00
        const match = date.match(/^(\d{4})?(\/\d{2})?(\/\d{2})?\s?(\d{2})?(:\d{2})?(:\d{2})?$/)

        if (match) {
          let offset = 0
          if (minoffset !== undefined) {
            offset = 60000 * (utils.timezoneOffset - minoffset)
          }

          const dateObj = new Date(Date.now() + offset)

          if (match[1] !== undefined) {
            dateObj.setFullYear(Number(match[1]), // full year
              match[2] !== undefined ? (Number(match[2].slice(1)) - 1) : 0, // month, zero-based
              match[3] !== undefined ? Number(match[3].slice(1)) : 1) // date
          }

          if (match[4] !== undefined) {
            dateObj.setHours(Number(match[4]), // hours
              match[5] !== undefined ? Number(match[5].slice(1)) : 0, // minutes
              match[6] !== undefined ? Number(match[6].slice(1)) : 0) // seconds
          }

          if (resetMs) {
            dateObj.setMilliseconds(0)
          }

          // Calculate timezone differences
          return new Date(dateObj.getTime() - offset)
        } else {
          return null
        }
      }

      // Parse dates to timestamps
      for (const range of ranges) {
        if (filterObj.queries[range]) {
          if (filterObj.queries[range].from) {
            const parsed = parseDate(filterObj.queries[range].from, minoffset, true)
            filterObj.queries[range].from = parsed ? Math.floor(parsed / 1000) : null
          }
          if (filterObj.queries[range].to) {
            const parsed = parseDate(filterObj.queries[range].to, minoffset, true)
            filterObj.queries[range].to = parsed ? Math.ceil(parsed / 1000) : null
          }
        }
      }

      // Query users table for user IDs
      if (filterObj.queries.user || filterObj.queries.exclude.user) {
        const usernames = []
        if (filterObj.queries.user) {
          usernames.push(...filterObj.queries.user)
        }
        if (filterObj.queries.exclude.user) {
          usernames.push(...filterObj.queries.exclude.user)
        }

        const uploaders = await db.table('users')
          .whereIn('username', usernames)
          .select('id', 'username')

        // If no matches, or mismatched results
        if (!uploaders || (uploaders.length !== usernames.length)) {
          const notFound = usernames.filter(username => {
            return !uploaders.find(uploader => uploader.username === username)
          })
          if (notFound) {
            throw new ClientError(`User${notFound.length === 1 ? '' : 's'} not found: ${notFound.join(', ')}.`)
          }
        }

        for (const uploader of uploaders) {
          if (filterObj.queries.user && filterObj.queries.user.includes(uploader.username)) {
            filterObj.uploaders.push(uploader)
          } else {
            filterObj.excludeUploaders.push(uploader)
          }
        }

        // Delete keys to avoid unexpected behavior
        delete filterObj.queries.user
        delete filterObj.queries.exclude.user
      }

      // Parse sort keys
      if (filterObj.queries.sort) {
        const allowed = [
          'expirydate',
          'id',
          'name',
          'original',
          'size',
          'timestamp'
        ]

        // Only allow sorting by 'albumid' when not listing album's uploads
        if (req.params.id === undefined) allowed.push('albumid')

        // Only allow sorting by 'ip' and 'userid' columns when listing all uploads
        if (all) allowed.push('ip', 'userid')

        for (const obQuery of filterObj.queries.sort) {
          const tmp = obQuery.toLowerCase().split(':')
          const column = sortObj.maps[tmp[0]] || tmp[0]

          if (!allowed.includes(column)) {
            // Alert users if using disallowed/missing columns
            throw new ClientError(`Column "${column}" cannot be used for sorting.\n\nTry the following instead:\n${allowed.join(', ')}`)
          }

          sortObj.parsed.push({
            column,
            order: (tmp[1] && /^d/.test(tmp[1])) ? 'desc' : 'asc',
            clause: sortObj.nullsLast.includes(column) ? 'nulls last' : '',
            cast: sortObj.casts[column] || null
          })
        }

        // Regular user threshold check
        if (!ismoderator && sortObj.parsed.length > MAX_SORT_KEYS) {
          throw new ClientError(`Users are only allowed to use ${MAX_SORT_KEYS} sort key${MAX_SORT_KEYS === 1 ? '' : 's'} at a time.`)
        }

        // Delete key to avoid unexpected behavior
        delete filterObj.queries.sort
      }

      // Parse is keys
      let isKeys = 0
      let isLast
      if (filterObj.queries.is || filterObj.queries.exclude.is) {
        for (const type of filterObj.typeIs) {
          const inQuery = filterObj.queries.is && filterObj.queries.is.includes(type)
          const inExclude = filterObj.queries.exclude.is && filterObj.queries.exclude.is.includes(type)

          // Prioritize exclude keys when both types found
          if (inQuery || inExclude) {
            filterObj.flags[`is${type}`] = inExclude ? false : inQuery
            if (isLast !== undefined && isLast !== filterObj.flags[`is${type}`]) {
              throw new ClientError('Cannot mix inclusion and exclusion type-is keys.')
            }
            isKeys++
            isLast = filterObj.flags[`is${type}`]
          }
        }

        // Delete keys to avoid unexpected behavior
        delete filterObj.queries.is
        delete filterObj.queries.exclude.is
      }

      // Regular user threshold check
      if (!ismoderator && isKeys > MAX_IS_KEYS) {
        throw new ClientError(`Users are only allowed to use ${MAX_IS_KEYS} type-is key${MAX_IS_KEYS === 1 ? '' : 's'} at a time.`)
      }
    }

    function filter () {
      // If listing all uploads
      if (all) {
        this.where(function () {
          // Filter uploads matching any of the supplied 'user' keys and/or NULL flag
          // Prioritze exclude keys when both types found
          this.orWhere(function () {
            if (filterObj.excludeUploaders.length) {
              this.whereNotIn('userid', filterObj.excludeUploaders.map(v => v.id))
            } else if (filterObj.uploaders.length) {
              this.orWhereIn('userid', filterObj.uploaders.map(v => v.id))
            }
            // Such overbearing logic for NULL values, smh...
            if ((filterObj.excludeUploaders.length && filterObj.flags.userNull !== false) ||
            (filterObj.uploaders.length && filterObj.flags.userNull) ||
            (!filterObj.excludeUploaders.length && !filterObj.uploaders.length && filterObj.flags.userNull)) {
              this.orWhereNull('userid')
            } else if (filterObj.flags.userNull === false) {
              this.whereNotNull('userid')
            }
          })

          // Filter uploads matching any of the supplied 'ip' keys and/or NULL flag
          // Same prioritization logic as above
          this.orWhere(function () {
            if (filterObj.queries.exclude.ip) {
              this.whereNotIn('ip', filterObj.queries.exclude.ip)
            } else if (filterObj.queries.ip) {
              this.orWhereIn('ip', filterObj.queries.ip)
            }
            // ...
            if ((filterObj.queries.exclude.ip && filterObj.flags.ipNull !== false) ||
            (filterObj.queries.ip && filterObj.flags.ipNull) ||
            (!filterObj.queries.exclude.ip && !filterObj.queries.ip && filterObj.flags.ipNull)) {
              this.orWhereNull('ip')
            } else if (filterObj.flags.ipNull === false) {
              this.whereNotNull('ip')
            }
          })
        })
      } else {
        // If not listing all uploads, list user's uploads
        this.where('userid', user.id)
      }

      // Then, refine using any of the supplied 'albumid' keys and/or NULL flag
      // Same prioritization logic as 'userid' and 'ip' above
      if (req.params.id === undefined) {
        this.andWhere(function () {
          if (filterObj.queries.exclude.albumid) {
            this.whereNotIn('albumid', filterObj.queries.exclude.albumid)
          } else if (filterObj.queries.albumid) {
            this.orWhereIn('albumid', filterObj.queries.albumid)
          }
          // ...
          if ((filterObj.queries.exclude.albumid && filterObj.flags.albumidNull !== false) ||
          (filterObj.queries.albumid && filterObj.flags.albumidNull) ||
          (!filterObj.queries.exclude.albumid && !filterObj.queries.albumid && filterObj.flags.albumidNull)) {
            this.orWhereNull('albumid')
          } else if (filterObj.flags.albumidNull === false) {
            this.whereNotNull('albumid')
          }
        })
      } else if (!all) {
        // If not listing all uploads, list uploads from user's album
        this.andWhere('albumid', req.params.id)
      }

      // Then, refine using the supplied 'date' ranges
      this.andWhere(function () {
        if (!filterObj.queries.date || (!filterObj.queries.date.from && !filterObj.queries.date.to)) return
        if (typeof filterObj.queries.date.from === 'number') {
          if (typeof filterObj.queries.date.to === 'number') {
            this.andWhereBetween('timestamp', [filterObj.queries.date.from, filterObj.queries.date.to])
          } else {
            this.andWhere('timestamp', '>=', filterObj.queries.date.from)
          }
        } else {
          this.andWhere('timestamp', '<=', filterObj.queries.date.to)
        }
      })

      // Then, refine using the supplied 'expiry' ranges
      this.andWhere(function () {
        if (!filterObj.queries.expiry || (!filterObj.queries.expiry.from && !filterObj.queries.expiry.to)) return
        if (typeof filterObj.queries.expiry.from === 'number') {
          if (typeof filterObj.queries.expiry.to === 'number') {
            this.andWhereBetween('expirydate', [filterObj.queries.expiry.from, filterObj.queries.expiry.to])
          } else {
            this.andWhere('expirydate', '>=', filterObj.queries.expiry.from)
          }
        } else {
          this.andWhere('expirydate', '<=', filterObj.queries.expiry.to)
        }
      })

      // Then, refine using type-is flags
      this.andWhere(function () {
        for (const type of filterObj.typeIs) {
          let func
          let operator
          if (filterObj.flags[`is${type}`] === true) {
            func = 'orWhere'
            operator = 'like'
          } else if (filterObj.flags[`is${type}`] === false) {
            func = 'andWhere'
            operator = 'not like'
          }

          if (func) {
            for (const pattern of utils[`${type}Exts`].map(ext => `%${ext}`)) {
              this[func]('name', operator, pattern)
            }
          }
        }
      })

      // Then, refine using the supplied keywords against their file names
      this.andWhere(function () {
        if (!filterObj.queries.text) return
        for (const pattern of filterObj.queries.text) {
          this.orWhereRaw('?? like ? escape ?', ['name', pattern, '\\'])
          this.orWhereRaw('?? like ? escape ?', ['original', pattern, '\\'])
        }
      })

      // Finally, refine using the supplied exclusions against their file names
      this.andWhere(function () {
        if (!filterObj.queries.exclude.text) return
        for (const pattern of filterObj.queries.exclude.text) {
          this.andWhereRaw('?? not like ? escape ?', ['name', pattern, '\\'])
          this.andWhereRaw('?? not like ? escape ?', ['original', pattern, '\\'])
        }
      })
    }

    // Query uploads count for pagination
    const count = await db.table('files')
      .where(filter)
      .count('id as count')
      .then(rows => rows[0].count)
    if (!count) return res.json({ success: true, files: [], count })

    let offset = Number(req.params.page)
    if (isNaN(offset)) offset = 0
    else if (offset < 0) offset = Math.max(0, Math.ceil(count / 25) + offset)

    const columns = ['id', 'name', 'original', 'userid', 'size', 'timestamp']
    if (temporaryUploads) columns.push('expirydate')
    if (!all ||
      filterObj.queries.albumid ||
      filterObj.queries.exclude.albumid ||
      filterObj.flags.albumidNull !== undefined) columns.push('albumid')

    // Only select IPs if we are listing all uploads
    if (all) columns.push('ip')

    // Build raw query for order by (sorting) operation
    let orderByRaw
    if (sortObj.parsed.length) {
      orderByRaw = sortObj.parsed.map(sort => {
        // Use Knex.raw() to sanitize user inputs
        if (sort.cast) {
          return db.raw(`cast (?? as ${sort.cast}) ${sort.order} ${sort.clause}`.trim(), sort.column)
        } else {
          return db.raw(`?? ${sort.order} ${sort.clause}`.trim(), sort.column)
        }
      }).join(', ')
    } else {
      orderByRaw = '`id` desc'
    }

    const files = await db.table('files')
      .where(filter)
      .orderByRaw(orderByRaw)
      .limit(25)
      .offset(25 * offset)
      .select(columns)

    if (!files.length) return res.json({ success: true, files, count, basedomain })

    for (const file of files) {
      file.extname = utils.extname(file.name)
      if (utils.mayGenerateThumb(file.extname)) {
        file.thumb = `thumbs/${file.name.slice(0, -file.extname.length)}.png`
      }
    }

    // If we queried albumid, query album names
    let albums = {}
    if (columns.includes('albumid')) {
      const albumids = files
        .map(file => file.albumid)
        .filter((v, i, a) => {
          return v !== null && v !== undefined && v !== '' && a.indexOf(v) === i
        })
      albums = await db.table('albums')
        .whereIn('id', albumids)
        .where('enabled', 1)
        .select('id', 'name')
        .then(rows => {
          // Build Object indexed by their IDs
          const obj = {}
          for (const row of rows) {
            obj[row.id] = row.name
          }
          return obj
        })
    }

    // If we are not listing all uploads, send response
    if (!all) return res.json({ success: true, files, count, albums, basedomain })

    // Otherwise proceed to querying usernames
    let usersTable = filterObj.uploaders
    if (!usersTable.length) {
      const userids = files
        .map(file => file.userid)
        .filter((v, i, a) => {
          return v !== null && v !== undefined && v !== '' && a.indexOf(v) === i
        })

      // If there are no uploads attached to a registered user, send response
      if (!userids.length) return res.json({ success: true, files, count, albums, basedomain })

      // Query usernames of user IDs from currently selected files
      usersTable = await db.table('users')
        .whereIn('id', userids)
        .select('id', 'username')
    }

    const users = {}
    for (const user of usersTable) {
      users[user.id] = user.username
    }

    await res.json({ success: true, files, count, users, albums, basedomain })
  } catch (error) {
    return apiErrorsHandler(error, req, res, next)
  }
}

module.exports = self