feat: experimental clamscan passthrough support

when enabled, passthrough scanning will be used for non-chunked uploads

upload processing will be significantly faster if scanning is required

config.sample.js

@@ -428,7 +428,11 @@ module.exports = {
           bypassTest: false
         },
         preference: 'clamdscan'
-      }
+      },
+
+      // Experimental .passthrough() support
+      // https://github.com/kylefarris/clamscan/tree/v2.1.2#passthrough
+      clamPassthrough: true
     },
 
     /*

controllers/uploadController.js

@@ -158,7 +158,9 @@ const executeMulter = multer({
           .then(name => cb(null, name))
           .catch(error => cb(error))
       }
-    }
+    },
+
+    clamscan: utils.clamscan
   })
 }).array('files[]')
 
@@ -338,8 +340,15 @@ self.actuallyUploadFiles = async (req, res, user, albumid, age) => {
   }
 
   if (utils.clamscan.instance) {
-    const scanResult = await self.scanFiles(req, user, infoMap)
-    if (scanResult) throw new ClientError(scanResult)
+    let scanResult
+    if (utils.clamscan.passthrough) {
+      scanResult = await self.assertPassthroughScans(req, user, infoMap)
+    } else {
+      scanResult = await self.scanFiles(req, user, infoMap)
+    }
+    if (scanResult) {
+      throw new ClientError(scanResult)
+    }
   }
 
   await self.stripTags(req, infoMap)
@@ -611,6 +620,42 @@ self.cleanUpChunks = async (uuid, onTimeout) => {
   delete chunksData[uuid]
 }
 
+self.assertPassthroughScans = async (req, user, infoMap) => {
+  const foundThreats = []
+  const unableToScan = []
+
+  for (const info of infoMap) {
+    if (info.data.clamscan) {
+      if (info.data.clamscan.isInfected) {
+        foundThreats.push(...info.data.clamscan.viruses)
+      } else if (info.data.clamscan.isInfected === null) {
+        unableToScan.push(info.data.filename)
+      }
+    } else {
+      unableToScan.push(info.data.filename)
+    }
+  }
+
+  let result = ''
+  if (foundThreats.length) {
+    const more = foundThreats.length > 1
+    result = `Threat${more ? 's' : ''} detected: ${foundThreats[0]}${more ? ', and more' : ''}.`
+  } else if (unableToScan.length) {
+    const more = unableToScan.length > 1
+    result = `Unable to scan: ${unableToScan[0]}${more ? ', and more' : ''}.`
+  }
+
+  if (result) {
+    // Unlink all files when at least one threat is found
+    // Should continue even when encountering errors
+    await Promise.all(infoMap.map(info =>
+      utils.unlinkFile(info.data.filename).catch(logger.error)
+    ))
+  }
+
+  return result
+}
+
 self.scanFiles = async (req, user, infoMap) => {
   if (user && utils.clamscan.groupBypass && perms.is(user, utils.clamscan.groupBypass)) {
     logger.debug(`[ClamAV]: Skipping ${infoMap.length} file(s), ${utils.clamscan.groupBypass} group bypass`)
@@ -619,7 +664,7 @@ self.scanFiles = async (req, user, infoMap) => {
 
   const foundThreats = []
   const unableToScan = []
-  const results = await Promise.all(infoMap.map(async info => {
+  const result = await Promise.all(infoMap.map(async info => {
     if (utils.clamscan.whitelistExtensions && utils.clamscan.whitelistExtensions.includes(info.data.extname)) {
       logger.debug(`[ClamAV]: Skipping ${info.data.filename}, extension whitelisted`)
       return
@@ -630,7 +675,7 @@ self.scanFiles = async (req, user, infoMap) => {
       return
     }
 
-    logger.debug(`[ClamAV]: Scanning ${info.data.filename}\u2026`)
+    logger.debug(`[ClamAV]: ${info.data.filename}: Scanning\u2026`)
     const response = await utils.clamscan.instance.isInfected(info.path)
     if (response.isInfected) {
       logger.log(`[ClamAV]: ${info.data.filename}: ${response.viruses.join(', ')}`)
@@ -652,7 +697,7 @@ self.scanFiles = async (req, user, infoMap) => {
     return 'An unexpected error occurred with ClamAV, please contact the site owner.'
   })
 
-  if (results) {
+  if (result) {
     // Unlink all files when at least one threat is found OR any errors occurred
     // Should continue even when encountering errors
     await Promise.all(infoMap.map(info =>
@@ -660,7 +705,7 @@ self.scanFiles = async (req, user, infoMap) => {
     ))
   }
 
-  return results
+  return result
 }
 
 self.stripTags = async (req, infoMap) => {

controllers/utils/multerStorage.js

@@ -2,6 +2,9 @@ const fs = require('fs')
 const path = require('path')
 const blake3 = require('blake3')
 const mkdirp = require('mkdirp')
+const logger = require('./../../logger')
+
+const REQUIRED_WEIGHT = 2
 
 function DiskStorage (opts) {
   this.getFilename = opts.filename
@@ -12,21 +15,36 @@ function DiskStorage (opts) {
   } else {
     this.getDestination = opts.destination
   }
+
+  this.clamscan = opts.clamscan
 }
 
 DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) {
   const that = this
 
+  // "weighted" callback, to be able to "await" multiple callbacks
+  let tempError = null
+  let tempObject = {}
+  let tempWeight = 0
+  const _cb = (err, result, weight = 1) => {
+    tempError = err
+    tempWeight += weight
+    tempObject = Object.assign(result, tempObject)
+    if (tempError || tempWeight >= REQUIRED_WEIGHT) {
+      cb(tempError, tempObject)
+    }
+  }
+
   that.getDestination(req, file, function (err, destination) {
-    if (err) return cb(err)
+    if (err) return _cb(err)
 
     that.getFilename(req, file, function (err, filename) {
-      if (err) return cb(err)
+      if (err) return _cb(err)
 
       const finalPath = path.join(destination, filename)
       const onerror = err => {
         hash.dispose()
-        cb(err)
+        _cb(err)
       }
 
       let outStream
@@ -53,24 +71,36 @@ DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) {
 
       if (file._isChunk) {
         file.stream.on('end', () => {
-          cb(null, {
+          _cb(null, {
             destination,
             filename,
             path: finalPath
-          })
+          }, 2)
         })
         file.stream.pipe(outStream, { end: false })
       } else {
         outStream.on('finish', () => {
-          cb(null, {
+          _cb(null, {
             destination,
             filename,
             path: finalPath,
             size: outStream.bytesWritten,
             hash: hash.digest('hex')
-          })
+          }, that.clamscan.passthrough ? 1 : 2)
         })
-        file.stream.pipe(outStream)
+
+        if (that.clamscan.passthrough) {
+          logger.debug(`[ClamAV]: ${filename}: Passthrough scanning\u2026`)
+          const clamStream = that.clamscan.instance.passthrough()
+          clamStream.on('scan-complete', result => {
+            _cb(null, {
+              clamscan: result
+            })
+          })
+          file.stream.pipe(clamStream).pipe(outStream)
+        } else {
+          file.stream.pipe(outStream)
+        }
       }
     })
   })

controllers/utilsController.js

@@ -22,7 +22,8 @@ const self = {
       config.uploads.scan.whitelistExtensions.length)
       ? config.uploads.scan.whitelistExtensions
       : null,
-    maxSize: (parseInt(config.uploads.scan.maxSize) * 1e6) || null
+    maxSize: (parseInt(config.uploads.scan.maxSize) * 1e6) || null,
+    passthrough: config.uploads.scan.clamPassthrough
   },
   gitHash: null,
   idSet: null,