From 86c26cb50c0bfd0d876369e2f79a65118de2e1ca Mon Sep 17 00:00:00 2001 From: Bobby Date: Sat, 23 Apr 2022 04:44:01 +0700 Subject: [PATCH] feat: some bypass support to passthrough scanning only usergroup and file extension bypass real file size can't be determined before passthrough scan, so there's no bypass by max file size please read the comments in sample config file refactored utils.clamscan into utils.scan --- config.sample.js | 18 +++++-- controllers/uploadController.js | 75 +++++++++++++++++++----------- controllers/utils/multerStorage.js | 24 ++++++---- controllers/utilsController.js | 10 ++-- lolisafe.js | 6 +-- 5 files changed, 85 insertions(+), 48 deletions(-) diff --git a/config.sample.js b/config.sample.js index 84f0084..131087c 100644 --- a/config.sample.js +++ b/config.sample.js @@ -408,7 +408,8 @@ module.exports = { '.mov', '.mkv' ], */ - // Make sure maxSize is no bigger than the max size you configured for your ClamAV + + // Make sure this doesn't exceed size limit in your ClamAV config maxSize: null, // Needs to be in MB // https://github.com/kylefarris/clamscan/tree/v2.1.2#getting-started @@ -440,9 +441,18 @@ module.exports = { preference: 'clamdscan' }, - // Experimental .passthrough() support - // Make sure StreamMaxLength option in ClamAV config is big enough - // https://github.com/kylefarris/clamscan/tree/v2.1.2#passthrough + /* + Experimental .passthrough() support. + https://github.com/kylefarris/clamscan/tree/v2.1.2#passthrough + + If enabled, StreamMaxLength in ClamAV config must be able to accommodate your + main "maxSize" option (not to be confused with "maxSize" in "scan" options group). + Final file size can't be determined before passthrough, + so file of all sizes will have to be scanned regardless. + + This will only passthrough scan non-chunked file uploads. + Chunked file uploads and URL uploads will still use the default scan method. + */ clamPassthrough: false }, diff --git a/controllers/uploadController.js b/controllers/uploadController.js index 72c0b93..f340b21 100644 --- a/controllers/uploadController.js +++ b/controllers/uploadController.js @@ -17,7 +17,8 @@ const logger = require('./../logger') const db = require('knex')(config.database) const self = { - onHold: new Set() + onHold: new Set(), + scanHelpers: {} } /** Preferences */ @@ -166,7 +167,8 @@ const executeMulter = multer({ } }, - clamscan: utils.clamscan + scan: utils.scan, + scanHelpers: self.scanHelpers }) }).array('files[]') @@ -302,8 +304,9 @@ self.upload = async (req, res, next) => { self.actuallyUploadFiles = async (req, res, user, albumid, age) => { const error = await new Promise(resolve => { + req._user = user return executeMulter(req, res, err => resolve(err)) - }) + }).finally(() => delete req._user) if (error) { const suppress = [ @@ -349,9 +352,9 @@ self.actuallyUploadFiles = async (req, res, user, albumid, age) => { throw new ClientError('Empty files are not allowed.') } - if (utils.clamscan.instance) { + if (utils.scan.instance) { let scanResult - if (utils.clamscan.passthrough) { + if (utils.scan.passthrough) { scanResult = await self.assertPassthroughScans(req, user, infoMap) } else { scanResult = await self.scanFiles(req, user, infoMap) @@ -461,7 +464,7 @@ self.actuallyUploadUrls = async (req, res, user, albumid, age) => { // If no errors encountered, clear cache of downloaded files downloaded.length = 0 - if (utils.clamscan.instance) { + if (utils.scan.instance) { const scanResult = await self.scanFiles(req, user, infoMap) if (scanResult) throw new ClientError(scanResult) } @@ -591,7 +594,7 @@ self.actuallyFinishChunks = async (req, res, user) => { infoMap.push({ path: destination, data }) })) - if (utils.clamscan.instance) { + if (utils.scan.instance) { const scanResult = await self.scanFiles(req, user, infoMap) if (scanResult) throw new ClientError(scanResult) } @@ -636,21 +639,43 @@ self.cleanUpChunks = async (uuid, onTimeout) => { /** Virus scanning (ClamAV) */ +self.scanHelpers.assertUserBypass = (user, filenames) => { + if (!user || !utils.scan.groupBypass) return false + if (!Array.isArray(filenames)) filenames = [filenames] + logger.debug(`[ClamAV]: ${filenames.join(', ')}: Skipped, uploaded by ${user.username} (${utils.scan.groupBypass})`) + return perms.is(user, utils.scan.groupBypass) +} + +self.scanHelpers.assertFileBypass = data => { + if (typeof data !== 'object' || !data.filename) return false + + const extname = data.extname || utils.extname(data.filename) + if (utils.scan.whitelistExtensions && utils.scan.whitelistExtensions.includes(extname)) { + logger.debug(`[ClamAV]: ${data.filename}: Skipped, extension whitelisted`) + return true + } + + if (Number.isFinite(data.size) && utils.scan.maxSize && data.size > utils.scan.maxSize) { + logger.debug(`[ClamAV]: ${data.filename}: Skipped, size ${data.size} > ${utils.scan.maxSize}`) + return true + } + + return false +} + self.assertPassthroughScans = async (req, user, infoMap) => { const foundThreats = [] const unableToScan = [] for (const info of infoMap) { - if (info.data.clamscan) { - if (info.data.clamscan.isInfected) { - logger.log(`[ClamAV]: ${info.data.filename}: ${info.data.clamscan.viruses.join(', ')}`) - foundThreats.push(...info.data.clamscan.viruses) - } else if (info.data.clamscan.isInfected === null) { + if (info.data.scan) { + if (info.data.scan.isInfected) { + logger.log(`[ClamAV]: ${info.data.filename}: ${info.data.scan.viruses.join(', ')}`) + foundThreats.push(...info.data.scan.viruses) + } else if (info.data.scan.isInfected === null) { logger.log(`[ClamAV]: ${info.data.filename}: Unable to scan`) unableToScan.push(info.data.filename) } - } else { - unableToScan.push(info.data.filename) } } @@ -675,26 +700,22 @@ self.assertPassthroughScans = async (req, user, infoMap) => { } self.scanFiles = async (req, user, infoMap) => { - if (user && utils.clamscan.groupBypass && perms.is(user, utils.clamscan.groupBypass)) { - logger.debug(`[ClamAV]: Skipping ${infoMap.length} file(s), ${utils.clamscan.groupBypass} group bypass`) + const filenames = infoMap.map(info => info.data.filename) + if (self.scanHelpers.assertUserBypass(user, filenames)) { return false } const foundThreats = [] const unableToScan = [] const result = await Promise.all(infoMap.map(async info => { - if (utils.clamscan.whitelistExtensions && utils.clamscan.whitelistExtensions.includes(info.data.extname)) { - logger.debug(`[ClamAV]: Skipping ${info.data.filename}, extension whitelisted`) - return - } - - if (utils.clamscan.maxSize && info.data.size > utils.clamscan.maxSize) { - logger.debug(`[ClamAV]: Skipping ${info.data.filename}, size ${info.data.size} > ${utils.clamscan.maxSize}`) - return - } + if (self.scanHelpers.assertFileBypass({ + filename: info.data.filename, + extname: info.data.extname, + size: info.data.size + })) return logger.debug(`[ClamAV]: ${info.data.filename}: Scanning\u2026`) - const response = await utils.clamscan.instance.isInfected(info.path) + const response = await utils.scan.instance.isInfected(info.path) if (response.isInfected) { logger.log(`[ClamAV]: ${info.data.filename}: ${response.viruses.join(', ')}`) foundThreats.push(...response.viruses) @@ -711,7 +732,7 @@ self.scanFiles = async (req, user, infoMap) => { return `Unable to scan: ${unableToScan[0]}${more ? ', and more' : ''}.` } }).catch(error => { - logger.error(`[ClamAV]: ${infoMap.map(info => info.data.filename).join(', ')}: ${error.toString()}`) + logger.error(`[ClamAV]: ${filenames.join(', ')}: ${error.toString()}`) return 'An unexpected error occurred with ClamAV, please contact the site owner.' }) diff --git a/controllers/utils/multerStorage.js b/controllers/utils/multerStorage.js index 044b70e..e08aff2 100644 --- a/controllers/utils/multerStorage.js +++ b/controllers/utils/multerStorage.js @@ -16,7 +16,8 @@ function DiskStorage (opts) { this.getDestination = opts.destination } - this.clamscan = opts.clamscan + this.scan = opts.scan + this.scanHelpers = opts.scanHelpers } DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) { @@ -49,6 +50,7 @@ DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) { let outStream let hash + let scanStream if (file._isChunk) { if (!file._chunksData.stream) { file._chunksData.stream = fs.createWriteStream(finalPath, { flags: 'a' }) @@ -64,6 +66,12 @@ DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) { outStream = fs.createWriteStream(finalPath) outStream.on('error', onerror) hash = blake3.createHash() + + if (that.scan.passthrough && + !that.scanHelpers.assertUserBypass(req._user, filename) && + !that.scanHelpers.assertFileBypass({ filename })) { + scanStream = that.scan.instance.passthrough() + } } file.stream.on('error', onerror) @@ -86,18 +94,16 @@ DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) { path: finalPath, size: outStream.bytesWritten, hash: hash.digest('hex') - }, that.clamscan.passthrough ? 1 : 2) + }, scanStream ? 1 : 2) }) - if (that.clamscan.passthrough) { + if (scanStream) { logger.debug(`[ClamAV]: ${filename}: Passthrough scanning\u2026`) - const clamStream = that.clamscan.instance.passthrough() - clamStream.on('scan-complete', result => { - _cb(null, { - clamscan: result - }, 1) + scanStream.on('error', onerror) + scanStream.on('scan-complete', scan => { + _cb(null, { scan }, 1) }) - file.stream.pipe(clamStream).pipe(outStream) + file.stream.pipe(scanStream).pipe(outStream) } else { file.stream.pipe(outStream) } diff --git a/controllers/utilsController.js b/controllers/utilsController.js index 29f3ae3..972ee75 100644 --- a/controllers/utilsController.js +++ b/controllers/utilsController.js @@ -14,7 +14,7 @@ const logger = require('./../logger') const db = require('knex')(config.database) const self = { - clamscan: { + scan: { instance: null, version: null, groupBypass: config.uploads.scan.groupBypass || null, @@ -693,12 +693,12 @@ self.stats = async (req, res, next) => { const time = si.time() const nodeUptime = process.uptime() - if (self.clamscan.instance) { + if (self.scan.instance) { try { - self.clamscan.version = await self.clamscan.instance.getVersion().then(s => s.trim()) + self.scan.version = await self.scan.instance.getVersion().then(s => s.trim()) } catch (error) { logger.error(error) - self.clamscan.version = 'Errored when querying version.' + self.scan.version = 'Errored when querying version.' } } @@ -706,7 +706,7 @@ self.stats = async (req, res, next) => { Platform: `${os.platform} ${os.arch}`, Distro: `${os.distro} ${os.release}`, Kernel: os.kernel, - Scanner: self.clamscan.version || 'N/A', + Scanner: self.scan.version || 'N/A', 'CPU Load': `${currentLoad.currentLoad.toFixed(1)}%`, 'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '), 'System Memory': { diff --git a/lolisafe.js b/lolisafe.js index 8efd620..f70969f 100644 --- a/lolisafe.js +++ b/lolisafe.js @@ -317,9 +317,9 @@ safe.use('/api', api) logger.error('Missing object config.uploads.scan.clamOptions (check config.sample.js)') process.exit(1) } - utils.clamscan.instance = await new NodeClam().init(config.uploads.scan.clamOptions) - utils.clamscan.version = await utils.clamscan.instance.getVersion().then(s => s.trim()) - logger.log(`Connection established with ${utils.clamscan.version}`) + utils.scan.instance = await new NodeClam().init(config.uploads.scan.clamOptions) + utils.scan.version = await utils.scan.instance.getVersion().then(s => s.trim()) + logger.log(`Connection established with ${utils.scan.version}`) } // Cache file identifiers