Switched to BLAKE3 for file hashing [MORE]

UPDATE YOUR CONFIG FILE IF YOU USE CHUNKED UPLOADS!
Read more about this at the end.

Added new dependency: blake3

Hashes will be created as the uploads are being written to disk.
With exception for chunked uploads!
For them specifically, their hashes will be created as they're being
rebuilt into a single file.
Should still be a lot better than the previous case where it had to
re-read the already written files.

To support that feature, added a new file
controllers/multerStorageController.js.
It's just a custom storage engine for Multer.

chunkSize option now allows setting max chunk size from config file.
Previously it was hardcoded to 95MB, but assuming you have paid
Cloudflare plans, you can actually have up to 500MB.

Also moved the option to be after maxSize and before urlMaxSize.
Made a lot more sense to me this way, as chunked uploads only work on
regular uploads.

Updated v1 version string and rebuilt client assets.

config.sample.js

@@ -206,6 +206,27 @@ module.exports = {
     */
     maxSize: '512MB',
 
+    /*
+      Chunk size for chunked uploads. Needs to be in MB.
+      If this is enabled, every files uploaded from the homepage uploader
+      will forcibly be chunked by the size specified in "default".
+      Users can configure the chunk size they want from the homepage uploader,
+      but you can force allowed max size of each chunk with "max".
+      Min size will always be 1MB.
+      Users will still be able to upload bigger files with the API
+      as long as they don't surpass the limit specified in the "maxSize" option above.
+      Once all chunks have been uploads, their total size
+      will be tested against the "maxSize" option again.
+      This option is mainly useful for hosters that use Cloudflare,
+      since Cloudflare limits upload size to 100MB on their Free plan.
+      https://support.cloudflare.com/hc/en-us/articles/200172516#h_51422705-42d0-450d-8eb1-5321dcadb5bc
+      NOTE: Set to falsy value to disable chunked uploads.
+    */
+    chunkSize: {
+      max: '95MB',
+      default: '25MB'
+    },
+
     /*
       Max file size allowed for upload by URLs. Needs to be in MB.
       NOTE: Set to falsy value to disable upload by URLs.
@@ -341,16 +362,6 @@ module.exports = {
     */
     storeIP: true,
 
-    /*
-      Chunk size for chunk uploads. Needs to be in MB.
-      If this is enabled, every files uploaded from the homepage uploader will forcibly be chunked
-      by the size specified in "chunkSize". People will still be able to upload bigger files with
-      the API as long as they don't surpass the limit specified in the "maxSize" option above.
-      Total size of the whole chunks will also later be checked against the "maxSize" option.
-      NOTE: Set to falsy value to disable chunked uploads.
-    */
-    chunkSize: '10MB',
-
     /*
       The length of the randomly generated identifier for uploaded files.
       If "force" is set to true, files will always use "default".

controllers/multerStorageController.js

@@ -0,0 +1,79 @@
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const crypto = require('crypto')
+const blake3 = require('blake3')
+const mkdirp = require('mkdirp')
+
+function getFilename (req, file, cb) {
+  // This won't be used since we use our own filename function.
+  crypto.randomBytes(16, function (err, raw) {
+    cb(err, err ? undefined : raw.toString('hex'))
+  })
+}
+
+function getDestination (req, file, cb) {
+  cb(null, os.tmpdir())
+}
+
+function DiskStorage (opts) {
+  this.getFilename = (opts.filename || getFilename)
+
+  if (typeof opts.destination === 'string') {
+    mkdirp.sync(opts.destination)
+    this.getDestination = function ($0, $1, cb) { cb(null, opts.destination) }
+  } else {
+    this.getDestination = (opts.destination || getDestination)
+  }
+}
+
+DiskStorage.prototype._handleFile = function _handleFile (req, file, cb) {
+  const that = this
+
+  that.getDestination(req, file, function (err, destination) {
+    if (err) return cb(err)
+
+    that.getFilename(req, file, function (err, filename) {
+      if (err) return cb(err)
+
+      const finalPath = path.join(destination, filename)
+      const outStream = fs.createWriteStream(finalPath)
+      file.stream.pipe(outStream)
+
+      let hash = null
+      if (!file._ischunk) {
+        hash = blake3.createHash()
+        file.stream.on('data', d => hash.update(d))
+        file.stream.on('error', err => {
+          hash.dispose()
+          return cb(err)
+        })
+      }
+
+      outStream.on('error', cb)
+      outStream.on('finish', function () {
+        cb(null, {
+          destination,
+          filename,
+          path: finalPath,
+          size: outStream.bytesWritten,
+          hash: hash && hash.digest('hex')
+        })
+      })
+    })
+  })
+}
+
+DiskStorage.prototype._removeFile = function _removeFile (req, file, cb) {
+  const path = file.path
+
+  delete file.destination
+  delete file.filename
+  delete file.path
+
+  fs.unlink(path, cb)
+}
+
+module.exports = function (opts) {
+  return new DiskStorage(opts)
+}

controllers/uploadController.js

@@ -1,10 +1,11 @@
-const crypto = require('crypto')
+const blake3 = require('blake3')
 const fetch = require('node-fetch')
 const fs = require('fs')
 const multer = require('multer')
 const path = require('path')
 const randomstring = require('randomstring')
 const searchQuery = require('search-query-parser')
+const multerStorage = require('./multerStorageController')
 const paths = require('./pathsController')
 const perms = require('./permissionController')
 const utils = require('./utilsController')
@@ -25,7 +26,9 @@ const urlMaxSizeBytes = parseInt(config.uploads.urlMaxSize) * 1e6
 
 const maxFilesPerUpload = 20
 
-const chunkedUploads = Boolean(config.uploads.chunkSize)
+const chunkedUploads = config.uploads.chunkSize &&
+  typeof config.uploads.chunkSize === 'object' &&
+  config.uploads.chunkSize.default
 const chunksData = {}
 //  Hard-coded min chunk size of 1 MB (e.g. 50 MB = max 50 chunks)
 const maxChunksCount = maxSize
@@ -84,34 +87,35 @@ const executeMulter = multer({
     else
       return cb(null, true)
   },
-  storage: multer.diskStorage({
+  storage: multerStorage({
     destination (req, file, cb) {
-      // If chunked uploads is disabled or the uploaded file is not a chunk
-      if (!chunkedUploads || (req.body.uuid === undefined && req.body.chunkindex === undefined))
-        return cb(null, paths.uploads)
+      // Is file a chunk!?
+      file._ischunk = chunkedUploads && req.body.uuid !== undefined && req.body.chunkindex !== undefined
 
+      if (file._ischunk)
         initChunks(req.body.uuid)
           .then(uuidDir => cb(null, uuidDir))
           .catch(error => {
             logger.error(error)
             return cb('Could not process the chunked upload. Try again?')
           })
+      else
+        return cb(null, paths.uploads)
     },
 
     filename (req, file, cb) {
-      // If chunked uploads is disabled or the uploaded file is not a chunk
-      if (!chunkedUploads || (req.body.uuid === undefined && req.body.chunkindex === undefined)) {
-        const length = self.parseFileIdentifierLength(req.headers.filelength)
-        return self.getUniqueRandomName(length, file.extname)
-          .then(name => cb(null, name))
-          .catch(error => cb(error))
-      }
-
+      if (file._ischunk) {
         // index.extension (i.e. 0, 1, ..., n - will prepend zeros depending on the amount of chunks)
         const digits = req.body.totalchunkcount !== undefined ? `${req.body.totalchunkcount - 1}`.length : 1
         const zeros = new Array(digits + 1).join('0')
         const name = (zeros + req.body.chunkindex).slice(-digits)
         return cb(null, name)
+      } else {
+        const length = self.parseFileIdentifierLength(req.headers.filelength)
+        return self.getUniqueRandomName(length, file.extname)
+          .then(name => cb(null, name))
+          .catch(error => cb(error))
+      }
     }
   })
 }).array('files[]')
@@ -452,7 +456,7 @@ self.actuallyFinishChunks = async (req, res, user) => {
 
       // Combine chunks
       const destination = path.join(paths.uploads, name)
-      await self.combineChunks(destination, file.uuid)
+      const hash = await self.combineChunks(destination, file.uuid)
 
       // Continue even when encountering errors
       await self.cleanUpChunks(file.uuid).catch(logger.error)
@@ -472,6 +476,7 @@ self.actuallyFinishChunks = async (req, res, user) => {
         extname: file.extname,
         mimetype: file.type || '',
         size: file.size,
+        hash,
         albumid,
         age: file.age
       }
@@ -504,15 +509,22 @@ self.actuallyFinishChunks = async (req, res, user) => {
 self.combineChunks = async (destination, uuid) => {
   let errorObj
   const writeStream = fs.createWriteStream(destination, { flags: 'a' })
+  const hash = blake3.createHash()
 
   try {
     chunksData[uuid].chunks.sort()
     for (const chunk of chunksData[uuid].chunks)
       await new Promise((resolve, reject) => {
-        fs.createReadStream(path.join(chunksData[uuid].root, chunk))
-          .on('error', error => reject(error))
-          .on('end', () => resolve())
-          .pipe(writeStream, { end: false })
+        const stream = fs.createReadStream(path.join(chunksData[uuid].root, chunk))
+        stream.pipe(writeStream, { end: false })
+
+        stream.on('data', d => hash.update(d))
+        stream.on('error', error => {
+          hash.dispose()
+          reject(error)
+        })
+
+        stream.on('end', () => resolve())
       })
   } catch (error) {
     errorObj = error
@@ -523,6 +535,9 @@ self.combineChunks = async (destination, uuid) => {
 
   // Re-throw error
   if (errorObj) throw errorObj
+
+  // Return hash
+  return hash.digest('hex')
 }
 
 self.cleanUpChunks = async (uuid) => {
@@ -602,17 +617,8 @@ self.storeFilesToDb = async (req, res, user, infoMap) => {
   const albumids = []
 
   await Promise.all(infoMap.map(async info => {
-    // Create hash of the file
-    const hash = await new Promise((resolve, reject) => {
-      const result = crypto.createHash('md5')
-      fs.createReadStream(info.path)
-        .on('error', error => reject(error))
-        .on('end', () => resolve(result.digest('hex')))
-        .on('data', data => result.update(data, 'utf8'))
-    })
-
     // Check if the file exists by checking its hash and size
-    const dbFile = await db.table('files')
+    const dbFile = info.data.hash && await db.table('files')
       .where(function () {
         if (user === undefined)
           this.whereNull('userid')
@@ -620,7 +626,7 @@ self.storeFilesToDb = async (req, res, user, infoMap) => {
           this.where('userid', user.id)
       })
       .where({
-        hash,
+        hash: info.data.hash,
         size: info.data.size
       })
       // Select expirydate to display expiration date of existing files as well
@@ -646,7 +652,7 @@ self.storeFilesToDb = async (req, res, user, infoMap) => {
       original: info.data.originalname,
       type: info.data.mimetype,
       size: info.data.size,
-      hash,
+      hash: info.data.hash,
       // Only disable if explicitly set to false in config
       ip: config.uploads.storeIP !== false ? req.ip : null,
       timestamp

package.json

@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "bcrypt": "^4.0.1",
+    "blake3": "^2.1.4",
     "body-parser": "^1.19.0",
     "clamdjs": "^1.0.2",
     "express": "^4.17.1",

src/js/home.js

@@ -19,7 +19,7 @@ const page = {
   private: null,
   enableUserAccounts: null,
   maxSize: null,
-  chunkSize: null,
+  chunkSizeConfig: null,
   temporaryUploadAges: null,
   fileIdentifierLength: null,
   stripTagsConfig: null,
@@ -27,14 +27,16 @@ const page = {
   // store album id that will be used with upload requests
   album: null,
 
-  parallelUploads: 2,
+  parallelUploads: null,
   previewImages: null,
   fileLength: null,
   uploadAge: null,
+  stripTags: null,
 
   maxSizeBytes: null,
   urlMaxSize: null,
   urlMaxSizeBytes: null,
+  chunkSize: null,
 
   tabs: [],
   activeTab: null,
@@ -157,9 +159,14 @@ page.checkIfPublic = () => {
 
     page.private = response.data.private
     page.enableUserAccounts = response.data.enableUserAccounts
+
     page.maxSize = parseInt(response.data.maxSize)
     page.maxSizeBytes = page.maxSize * 1e6
-    page.chunkSize = parseInt(response.data.chunkSize)
+    page.chunkSizeConfig = {
+      max: (response.data.chunkSize && parseInt(response.data.chunkSize.max)) || 95,
+      default: response.data.chunkSize && parseInt(response.data.chunkSize.default)
+    }
+
     page.temporaryUploadAges = response.data.temporaryUploadAges
     page.fileIdentifierLength = response.data.fileIdentifierLength
     page.stripTagsConfig = response.data.stripTags
@@ -754,11 +761,12 @@ page.createAlbum = () => {
 
 page.prepareUploadConfig = () => {
   const fallback = {
-    chunkSize: page.chunkSize,
-    parallelUploads: page.parallelUploads
+    chunkSize: page.chunkSizeConfig.default,
+    parallelUploads: 2
   }
 
-  const temporaryUploadAges = Array.isArray(page.temporaryUploadAges) && page.temporaryUploadAges.length
+  const temporaryUploadAges = Array.isArray(page.temporaryUploadAges) &&
+    page.temporaryUploadAges.length
   const fileIdentifierLength = page.fileIdentifierLength &&
     typeof page.fileIdentifierLength.min === 'number' &&
     typeof page.fileIdentifierLength.max === 'number'
@@ -802,11 +810,11 @@ page.prepareUploadConfig = () => {
       disabled: page.stripTagsConfig && page.stripTagsConfig.force
     },
     chunkSize: {
-      display: !isNaN(page.chunkSize),
+      display: Boolean(page.chunkSizeConfig.default),
       label: 'Upload chunk size (MB)',
       number: {
         min: 1,
-        max: 95,
+        max: page.chunkSizeConfig.max,
         suffix: ' MB',
         round: true
       },
@@ -895,7 +903,7 @@ page.prepareUploadConfig = () => {
         value = conf.value
       } else if (conf.number !== undefined) {
         const parsed = parseInt(localStorage[lsKeys[key]])
-        if (!isNaN(parsed))
+        if (!isNaN(parsed) && parsed <= conf.number.max && parsed >= conf.number.min)
           value = parsed
       } else {
         const stored = localStorage[lsKeys[key]]
@@ -911,6 +919,8 @@ page.prepareUploadConfig = () => {
         conf.valueHandler(value)
       else if (value !== undefined)
         page[key] = value
+      else if (fallback[key] !== undefined)
+        page[key] = fallback[key]
     }
 
     let control

src/versions.json

@@ -1,5 +1,5 @@
 {
-  "1": "1590617686",
+  "1": "1590695426",
   "2": "1589010026",
   "3": "1581416390",
   "4": "1581416390",

yarn.lock

@@ -788,6 +788,11 @@ bl@^4.0.1:
     inherits "^2.0.4"
     readable-stream "^3.4.0"
 
+blake3@^2.1.4:
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/blake3/-/blake3-2.1.4.tgz#78117bc9e80941097fdf7d03e897a9ee595ecd62"
+  integrity sha512-70hmx0lPd6zmtNwxPT4/1P0pqaEUlTJ0noUBvCXPLfMpN0o8PPaK3q7ZlpRIyhrqcXxeMAJSowNm/L9oi/x1XA==
+
 body-parser@1.19.0, body-parser@^1.19.0:
   version "1.19.0"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a"