Updates

* Faster upload response. Back-end will no longer wait for album timestamps to be updated before sending out response. * Added a simple thumbnail generation script at scripts/thumbs.js. You can use this to generate thumbnails for existing files before enabling the option in config.js. * Various other code improvements.
2025-01-18 17:21:33 +00:00 · 2018-05-12 21:01:14 +07:00 · 2018-05-12 21:01:14 +07:00 · 7f23734d67
commit 7f23734d67
parent 4d634196c5
5 changed files with 173 additions and 105 deletions
--- a/controllers/albumsController.js
+++ b/controllers/albumsController.js
@ -297,9 +297,9 @@ albumsController.get = async (req, res, next) => {
  for (const file of files) {
    file.file = `${config.domain}/${file.name}`

-    const ext = path.extname(file.name).toLowerCase()
-    if ((config.uploads.generateThumbs.image && utils.imageExtensions.includes(ext)) || (config.uploads.generateThumbs.video && utils.videoExtensions.includes(ext))) {
-      file.thumb = `${config.domain}/thumbs/${file.name.slice(0, -ext.length)}.png`
+    const extname = path.extname(file.name).toLowerCase()
+    if (utils.mayGenerateThumb(extname)) {
+      file.thumb = `${config.domain}/thumbs/${file.name.slice(0, -extname.length)}.png`
    }
  }

--- a/controllers/uploadController.js
+++ b/controllers/uploadController.js
@ -504,63 +504,51 @@ uploadsController.formatInfoMap = (req, res, user, infoMap) => {
 }

 uploadsController.processFilesForDisplay = async (req, res, files, existingFiles) => {
-  const basedomain = config.domain
-  let albumSuccess = true
-  let mappedFiles
+  const responseFiles = []

  if (files.length) {
    // Insert new files to DB
    await db.table('files').insert(files)

-    // Push existing files to array for response
-    for (const efile of existingFiles) {
-      files.push(efile)
-    }
-
-    const albumids = []
    for (const file of files) {
-      const ext = path.extname(file.name).toLowerCase()
-      if ((config.uploads.generateThumbs.image && utils.imageExtensions.includes(ext)) || (config.uploads.generateThumbs.video && utils.videoExtensions.includes(ext))) {
-        file.thumb = `${basedomain}/thumbs/${file.name.slice(0, -ext.length)}.png`
-        utils.generateThumbs(file)
-      }
-      if (file.albumid && !albumids.includes(file.albumid)) {
-        albumids.push(file.albumid)
-      }
+      responseFiles.push(file)
    }
-
-    if (albumids.length) {
-      await db.table('albums')
-        .whereIn('id', albumids)
-        .update('editedAt', Math.floor(Date.now() / 1000))
-        .catch(error => {
-          console.error(error)
-          albumSuccess = false
-        })
-    }
-
-    mappedFiles = files.map(file => {
-      return {
-        name: file.name,
-        size: file.size,
-        url: `${basedomain}/${file.name}`
-      }
-    })
-  } else {
-    mappedFiles = existingFiles.map(file => {
-      return {
-        name: file.name,
-        size: file.size,
-        url: `${basedomain}/${file.name}`
-      }
-    })
  }

-  return res.json({
-    success: albumSuccess,
-    description: albumSuccess ? null : 'Warning: Album may not have been properly updated.',
-    files: mappedFiles
+  if (existingFiles.length) {
+    for (const file of existingFiles) {
+      responseFiles.push(file)
+    }
+  }
+
+  // We send response first before generating thumbnails and updating album timestamps
+  res.json({
+    success: true,
+    files: responseFiles.map(file => {
+      return {
+        name: file.name,
+        size: file.size,
+        url: `${config.domain}/${file.name}`
+      }
+    })
  })
+
+  const albumids = []
+  for (const file of files) {
+    if (file.albumid && !albumids.includes(file.albumid)) {
+      albumids.push(file.albumid)
+    }
+    if (utils.mayGenerateThumb(path.extname(file.name).toLowerCase())) {
+      utils.generateThumbs(file.name)
+    }
+  }
+
+  if (albumids.length) {
+    await db.table('albums')
+      .whereIn('id', albumids)
+      .update('editedAt', Math.floor(Date.now() / 1000))
+      .catch(console.error)
+  }
 }

 uploadsController.delete = async (req, res) => {
@ -624,12 +612,10 @@ uploadsController.list = async (req, res) => {

  for (const file of files) {
    file.file = `${basedomain}/${file.name}`
-    file.date = new Date(file.timestamp * 1000)
-    file.date = utils.getPrettyDate(file.date)
+    file.date = utils.getPrettyDate(new Date(file.timestamp * 1000))
    file.size = utils.getPrettyBytes(parseInt(file.size))

    file.album = ''
-
    if (file.albumid !== undefined) {
      for (const album of albums) {
        if (file.albumid === album.id) {
@ -646,16 +632,9 @@ uploadsController.list = async (req, res) => {
    }

    file.extname = path.extname(file.name).toLowerCase()
-    const isVideoExt = utils.videoExtensions.includes(file.extname)
-    const isImageExt = utils.imageExtensions.includes(file.extname)
-
-    if ((!isVideoExt && !isImageExt) ||
-      (isVideoExt && config.uploads.generateThumbs.video !== true) ||
-      (isImageExt && config.uploads.generateThumbs.image !== true)) {
-      continue
+    if (utils.mayGenerateThumb(file.extname)) {
+      file.thumb = `${basedomain}/thumbs/${file.name.slice(0, -file.extname.length)}.png`
    }
-
-    file.thumb = `${basedomain}/thumbs/${file.name.slice(0, -file.extname.length)}.png`
  }

  // If we are a normal user, send response
--- a/controllers/utilsController.js
+++ b/controllers/utilsController.js
@ -63,47 +63,56 @@ utilsController.authorize = async (req, res) => {
  res.status(401).json({ success: false, description: 'Invalid token.' })
 }

-utilsController.generateThumbs = (file, basedomain) => {
-  const extname = path.extname(file.name).toLowerCase()
-  if (!utilsController.mayGenerateThumb(extname)) { return }
+utilsController.generateThumbs = (name, force) => {
+  return new Promise(resolve => {
+    const extname = path.extname(name).toLowerCase()
+    const thumbname = path.join(thumbsDir, name.slice(0, -extname.length) + '.png')
+    fs.access(thumbname, error => {
+      if (error && error.code !== 'ENOENT') {
+        console.error(error)
+        return resolve(false)
+      }

-  const thumbname = path.join(thumbsDir, file.name.slice(0, -extname.length) + '.png')
-  fs.access(thumbname, error => {
-    // Only make thumbnail if it does not exist (ENOENT)
-    if (!error || error.code !== 'ENOENT') { return }
+      // Only make thumbnail if it does not exist (ENOENT)
+      if (!error && !force) { return resolve(true) }

-    // If image extension
-    if (utilsController.imageExtensions.includes(extname)) {
-      const size = { width: 200, height: 200 }
-      return gm(path.join(__dirname, '..', config.uploads.folder, file.name))
-        .resize(size.width, size.height + '>')
-        .gravity('Center')
-        .extent(size.width, size.height)
-        .background('transparent')
-        .write(thumbname, error => {
-          if (error) {
-            console.error(`${file.name}: ${error.message.trim()}`)
+      // If image extension
+      if (utilsController.imageExtensions.includes(extname)) {
+        const size = { width: 200, height: 200 }
+        return gm(path.join(__dirname, '..', config.uploads.folder, name))
+          .resize(size.width, size.height + '>')
+          .gravity('Center')
+          .extent(size.width, size.height)
+          .background('transparent')
+          .write(thumbname, error => {
+            if (!error) { return resolve(true) }
+            console.error(`${name}: ${error.message.trim()}`)
            fs.symlink(thumbUnavailable, thumbname, error => {
              if (error) { console.error(error) }
+              resolve(!error)
            })
-          }
-        })
-    }
+          })
+      }

-    // Otherwise video extension
-    ffmpeg(path.join(__dirname, '..', config.uploads.folder, file.name))
-      .thumbnail({
-        timestamps: ['1%'],
-        filename: '%b.png',
-        folder: path.join(__dirname, '..', config.uploads.folder, 'thumbs'),
-        size: '200x?'
-      })
-      .on('error', error => {
-        console.log(`${file.name}: ${error.message}`)
-        fs.symlink(thumbUnavailable, thumbname, error => {
-          if (error) { console.error(error) }
+      // Otherwise video extension
+      ffmpeg(path.join(__dirname, '..', config.uploads.folder, name))
+        .thumbnail({
+          timestamps: ['1%'],
+          filename: '%b.png',
+          folder: path.join(__dirname, '..', config.uploads.folder, 'thumbs'),
+          size: '200x?'
        })
-      })
+        .on('error', error => {
+          console.log(`${name}: ${error.message}`)
+          fs.symlink(thumbUnavailable, thumbname, error => {
+            if (error) { console.error(error) }
+            resolve(!error)
+          })
+        })
+        .on('end', () => {
+          resolve(true)
+        })
+    })
  })
 }

--- a/routes/album.js
+++ b/routes/album.js
@ -42,17 +42,18 @@ routes.get('/a/:identifier', async (req, res, next) => {
  for (const file of files) {
    file.file = `${basedomain}/${file.name}`
    file.size = utils.getPrettyBytes(parseInt(file.size))
+
    file.extname = path.extname(file.name).toLowerCase()
+    if (utils.mayGenerateThumb(file.extname)) {
+      file.thumb = `${basedomain}/thumbs/${file.name.slice(0, -file.extname.length)}.png`

-    if (!utils.mayGenerateThumb(file.extname)) { continue }
-    file.thumb = `${basedomain}/thumbs/${file.name.slice(0, -file.extname.length)}.png`
-
-    /*
-      If thumbnail for album is still not set, do it.
-      A potential improvement would be to let the user upload a specific image as an album cover
-      since embedding the first image could potentially result in nsfw content when pasting links.
-    */
-    if (thumb === '') { thumb = file.thumb }
+      /*
+        If thumbnail for album is still not set, do it.
+        A potential improvement would be to let the user upload a specific image as an album cover
+        since embedding the first image could potentially result in nsfw content when pasting links.
+      */
+      if (thumb === '') { thumb = file.thumb }
+    }
  }

  return res.render('album', {
--- a/scripts/thumbs.js
+++ b/scripts/thumbs.js
@ -0,0 +1,79 @@
+const fs = require('fs')
+const path = require('path')
+const utils = require('./../controllers/utilsController')
+
+const thumbs = {
+  mode: null,
+  force: null
+}
+
+thumbs.mayGenerateThumb = extname => {
+  return ([1, 3].includes(thumbs.mode) && utils.imageExtensions.includes(extname)) ||
+    ([2, 3].includes(thumbs.mode) && utils.videoExtensions.includes(extname))
+}
+
+thumbs.getFiles = directory => {
+  return new Promise((resolve, reject) => {
+    fs.readdir(directory, async (error, names) => {
+      if (error) { return reject(error) }
+      const files = []
+      await Promise.all(names.map(name => {
+        return new Promise((resolve, reject) => {
+          fs.stat(path.join(directory, name), (error, stat) => {
+            if (error) { return reject(error) }
+            if (stat.isFile() && !name.startsWith('.')) { files.push(name) }
+            resolve()
+          })
+        })
+      }))
+      resolve(files)
+    })
+  })
+}
+
+thumbs.do = async () => {
+  const args = process.argv.slice(2)
+
+  thumbs.mode = parseInt(args[0])
+  thumbs.force = parseInt(args[1])
+  if ((isNaN(thumbs.mode) || ![1, 2, 3].includes(thumbs.mode)) ||
+  (!isNaN(thumbs.force) && ![0, 1].includes(thumbs.force))) {
+    console.log('Usage : node THIS_FILE <mode=1|2|3> [force=0|1]')
+    console.log('mode  : 1 = images only, 2 = videos only, 3 = both images and videos')
+    console.log('force : 0 = no force (default), 1 = overwrite existing thumbnails')
+    return
+  }
+
+  const uploadsDir = path.join(__dirname, '..', 'uploads')
+  const thumbsDir = path.join(uploadsDir, 'thumbs')
+  const _uploads = await thumbs.getFiles(uploadsDir)
+
+  let _thumbs = await thumbs.getFiles(thumbsDir)
+  _thumbs = _thumbs.map(_thumb => {
+    const extname = path.extname(_thumb)
+    return _thumb.slice(0, -extname.length)
+  })
+
+  await new Promise((resolve, reject) => {
+    const generate = async i => {
+      const _upload = _uploads[i]
+      if (!_upload) { return resolve() }
+
+      const extname = path.extname(_upload)
+      const basename = _upload.slice(0, -extname.length)
+
+      if (_thumbs.includes(basename) && !thumbs.force) {
+        console.log(`${_upload}: thumb exists.`)
+      } else if (!thumbs.mayGenerateThumb(extname)) {
+        console.log(`${_upload}: extension skipped.`)
+      } else {
+        const generated = await utils.generateThumbs(_upload, thumbs.force)
+        console.log(`${_upload}: ${String(generated)}`)
+      }
+      generate(i + 1)
+    }
+    generate(0)
+  })
+}
+
+thumbs.do()