feat: some bypass support to passthrough scanning

only usergroup and file extension bypass

real file size can't be determined before passthrough scan,
so there's no bypass by max file size
please read the comments in sample config file

refactored utils.clamscan into utils.scan

config.sample.js

@@ -408,7 +408,8 @@ module.exports = {
         '.mov',
         '.mkv'
       ], */
-      // Make sure maxSize is no bigger than the max size you configured for your ClamAV
+
+      // Make sure this doesn't exceed size limit in your ClamAV config
       maxSize: null, // Needs to be in MB
 
       // https://github.com/kylefarris/clamscan/tree/v2.1.2#getting-started
@@ -440,9 +441,18 @@ module.exports = {
         preference: 'clamdscan'
       },
 
-      // Experimental .passthrough() support
-      // Make sure StreamMaxLength option in ClamAV config is big enough
-      // https://github.com/kylefarris/clamscan/tree/v2.1.2#passthrough
+      /*
+        Experimental .passthrough() support.
+        https://github.com/kylefarris/clamscan/tree/v2.1.2#passthrough
+
+        If enabled, StreamMaxLength in ClamAV config must be able to accommodate your
+        main "maxSize" option (not to be confused with "maxSize" in "scan" options group).
+        Final file size can't be determined before passthrough,
+        so file of all sizes will have to be scanned regardless.
+
+        This will only passthrough scan non-chunked file uploads.
+        Chunked file uploads and URL uploads will still use the default scan method.
+      */
       clamPassthrough: false
     },
 

controllers/uploadController.js

@@ -17,7 +17,8 @@ const logger = require('./../logger')
 const db = require('knex')(config.database)
 
 const self = {
-  onHold: new Set()
+  onHold: new Set(),
+  scanHelpers: {}
 }
 
 /** Preferences */
@@ -166,7 +167,8 @@ const executeMulter = multer({
       }
     },
 
-    clamscan: utils.clamscan
+    scan: utils.scan,
+    scanHelpers: self.scanHelpers
   })
 }).array('files[]')
 
@@ -302,8 +304,9 @@ self.upload = async (req, res, next) => {
 
 self.actuallyUploadFiles = async (req, res, user, albumid, age) => {
   const error = await new Promise(resolve => {
+    req._user = user
     return executeMulter(req, res, err => resolve(err))
-  })
+  }).finally(() => delete req._user)
 
   if (error) {
     const suppress = [
@@ -349,9 +352,9 @@ self.actuallyUploadFiles = async (req, res, user, albumid, age) => {
     throw new ClientError('Empty files are not allowed.')
   }
 
-  if (utils.clamscan.instance) {
+  if (utils.scan.instance) {
     let scanResult
-    if (utils.clamscan.passthrough) {
+    if (utils.scan.passthrough) {
       scanResult = await self.assertPassthroughScans(req, user, infoMap)
     } else {
       scanResult = await self.scanFiles(req, user, infoMap)
@@ -461,7 +464,7 @@ self.actuallyUploadUrls = async (req, res, user, albumid, age) => {
     // If no errors encountered, clear cache of downloaded files
     downloaded.length = 0
 
-    if (utils.clamscan.instance) {
+    if (utils.scan.instance) {
       const scanResult = await self.scanFiles(req, user, infoMap)
       if (scanResult) throw new ClientError(scanResult)
     }
@@ -591,7 +594,7 @@ self.actuallyFinishChunks = async (req, res, user) => {
       infoMap.push({ path: destination, data })
     }))
 
-    if (utils.clamscan.instance) {
+    if (utils.scan.instance) {
       const scanResult = await self.scanFiles(req, user, infoMap)
       if (scanResult) throw new ClientError(scanResult)
     }
@@ -636,21 +639,43 @@ self.cleanUpChunks = async (uuid, onTimeout) => {
 
 /** Virus scanning (ClamAV) */
 
+self.scanHelpers.assertUserBypass = (user, filenames) => {
+  if (!user || !utils.scan.groupBypass) return false
+  if (!Array.isArray(filenames)) filenames = [filenames]
+  logger.debug(`[ClamAV]: ${filenames.join(', ')}: Skipped, uploaded by ${user.username} (${utils.scan.groupBypass})`)
+  return perms.is(user, utils.scan.groupBypass)
+}
+
+self.scanHelpers.assertFileBypass = data => {
+  if (typeof data !== 'object' || !data.filename) return false
+
+  const extname = data.extname || utils.extname(data.filename)
+  if (utils.scan.whitelistExtensions && utils.scan.whitelistExtensions.includes(extname)) {
+    logger.debug(`[ClamAV]: ${data.filename}: Skipped, extension whitelisted`)
+    return true
+  }
+
+  if (Number.isFinite(data.size) && utils.scan.maxSize && data.size > utils.scan.maxSize) {
+    logger.debug(`[ClamAV]: ${data.filename}: Skipped, size ${data.size} > ${utils.scan.maxSize}`)
+    return true
+  }
+
+  return false
+}
+
 self.assertPassthroughScans = async (req, user, infoMap) => {
   const foundThreats = []
   const unableToScan = []
 
   for (const info of infoMap) {
-    if (info.data.clamscan) {
-      if (info.data.clamscan.isInfected) {
-        logger.log(`[ClamAV]: ${info.data.filename}: ${info.data.clamscan.viruses.join(', ')}`)
-        foundThreats.push(...info.data.clamscan.viruses)
-      } else if (info.data.clamscan.isInfected === null) {
+    if (info.data.scan) {
+      if (info.data.scan.isInfected) {
+        logger.log(`[ClamAV]: ${info.data.filename}: ${info.data.scan.viruses.join(', ')}`)
+        foundThreats.push(...info.data.scan.viruses)
+      } else if (info.data.scan.isInfected === null) {
         logger.log(`[ClamAV]: ${info.data.filename}: Unable to scan`)
         unableToScan.push(info.data.filename)
       }
-    } else {
-      unableToScan.push(info.data.filename)
     }
   }
 
@@ -675,26 +700,22 @@ self.assertPassthroughScans = async (req, user, infoMap) => {
 }
 
 self.scanFiles = async (req, user, infoMap) => {
-  if (user && utils.clamscan.groupBypass && perms.is(user, utils.clamscan.groupBypass)) {
-    logger.debug(`[ClamAV]: Skipping ${infoMap.length} file(s), ${utils.clamscan.groupBypass} group bypass`)
+  const filenames = infoMap.map(info => info.data.filename)
+  if (self.scanHelpers.assertUserBypass(user, filenames)) {
     return false
   }
 
   const foundThreats = []
   const unableToScan = []
   const result = await Promise.all(infoMap.map(async info => {
-    if (utils.clamscan.whitelistExtensions && utils.clamscan.whitelistExtensions.includes(info.data.extname)) {
-      logger.debug(`[ClamAV]: Skipping ${info.data.filename}, extension whitelisted`)
-      return
-    }
-
-    if (utils.clamscan.maxSize && info.data.size > utils.clamscan.maxSize) {
-      logger.debug(`[ClamAV]: Skipping ${info.data.filename}, size ${info.data.size} > ${utils.clamscan.maxSize}`)
-      return
-    }
+    if (self.scanHelpers.assertFileBypass({
+      filename: info.data.filename,
+      extname: info.data.extname,
+      size: info.data.size
+    })) return
 
     logger.debug(`[ClamAV]: ${info.data.filename}: Scanning\u2026`)
-    const response = await utils.clamscan.instance.isInfected(info.path)
+    const response = await utils.scan.instance.isInfected(info.path)
     if (response.isInfected) {
       logger.log(`[ClamAV]: ${info.data.filename}: ${response.viruses.join(', ')}`)
       foundThreats.push(...response.viruses)
@@ -711,7 +732,7 @@ self.scanFiles = async (req, user, infoMap) => {
       return `Unable to scan: ${unableToScan[0]}${more ? ', and more' : ''}.`
     }
   }).catch(error => {
-    logger.error(`[ClamAV]: ${infoMap.map(info => info.data.filename).join(', ')}: ${error.toString()}`)
+    logger.error(`[ClamAV]: ${filenames.join(', ')}: ${error.toString()}`)
     return 'An unexpected error occurred with ClamAV, please contact the site owner.'
   })
 

controllers/utils/multerStorage.js

@@ -16,7 +16,8 @@ function DiskStorage (opts) {
     this.getDestination = opts.destination
   }
 
-  this.clamscan = opts.clamscan
+  this.scan = opts.scan
+  this.scanHelpers = opts.scanHelpers
 }
 
 DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) {
@@ -49,6 +50,7 @@ DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) {
 
       let outStream
       let hash
+      let scanStream
       if (file._isChunk) {
         if (!file._chunksData.stream) {
           file._chunksData.stream = fs.createWriteStream(finalPath, { flags: 'a' })
@@ -64,6 +66,12 @@ DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) {
         outStream = fs.createWriteStream(finalPath)
         outStream.on('error', onerror)
         hash = blake3.createHash()
+
+        if (that.scan.passthrough &&
+          !that.scanHelpers.assertUserBypass(req._user, filename) &&
+          !that.scanHelpers.assertFileBypass({ filename })) {
+          scanStream = that.scan.instance.passthrough()
+        }
       }
 
       file.stream.on('error', onerror)
@@ -86,18 +94,16 @@ DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) {
             path: finalPath,
             size: outStream.bytesWritten,
             hash: hash.digest('hex')
-          }, that.clamscan.passthrough ? 1 : 2)
+          }, scanStream ? 1 : 2)
         })
 
-        if (that.clamscan.passthrough) {
+        if (scanStream) {
           logger.debug(`[ClamAV]: ${filename}: Passthrough scanning\u2026`)
-          const clamStream = that.clamscan.instance.passthrough()
-          clamStream.on('scan-complete', result => {
-            _cb(null, {
-              clamscan: result
-            }, 1)
+          scanStream.on('error', onerror)
+          scanStream.on('scan-complete', scan => {
+            _cb(null, { scan }, 1)
           })
-          file.stream.pipe(clamStream).pipe(outStream)
+          file.stream.pipe(scanStream).pipe(outStream)
         } else {
           file.stream.pipe(outStream)
         }

controllers/utilsController.js

@@ -14,7 +14,7 @@ const logger = require('./../logger')
 const db = require('knex')(config.database)
 
 const self = {
-  clamscan: {
+  scan: {
     instance: null,
     version: null,
     groupBypass: config.uploads.scan.groupBypass || null,
@@ -693,12 +693,12 @@ self.stats = async (req, res, next) => {
         const time = si.time()
         const nodeUptime = process.uptime()
 
-        if (self.clamscan.instance) {
+        if (self.scan.instance) {
           try {
-            self.clamscan.version = await self.clamscan.instance.getVersion().then(s => s.trim())
+            self.scan.version = await self.scan.instance.getVersion().then(s => s.trim())
           } catch (error) {
             logger.error(error)
-            self.clamscan.version = 'Errored when querying version.'
+            self.scan.version = 'Errored when querying version.'
           }
         }
 
@@ -706,7 +706,7 @@ self.stats = async (req, res, next) => {
           Platform: `${os.platform} ${os.arch}`,
           Distro: `${os.distro} ${os.release}`,
           Kernel: os.kernel,
-          Scanner: self.clamscan.version || 'N/A',
+          Scanner: self.scan.version || 'N/A',
           'CPU Load': `${currentLoad.currentLoad.toFixed(1)}%`,
           'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '),
           'System Memory': {

lolisafe.js

@@ -317,9 +317,9 @@ safe.use('/api', api)
         logger.error('Missing object config.uploads.scan.clamOptions (check config.sample.js)')
         process.exit(1)
       }
-      utils.clamscan.instance = await new NodeClam().init(config.uploads.scan.clamOptions)
-      utils.clamscan.version = await utils.clamscan.instance.getVersion().then(s => s.trim())
-      logger.log(`Connection established with ${utils.clamscan.version}`)
+      utils.scan.instance = await new NodeClam().init(config.uploads.scan.clamOptions)
+      utils.scan.version = await utils.scan.instance.getVersion().then(s => s.trim())
+      logger.log(`Connection established with ${utils.scan.version}`)
     }
 
     // Cache file identifiers