Mirror/filesafe
1
0
Fork 0
mirror of https://github.com/BobbyWibowo/lolisafe.git synced 2025-02-22 13:19:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: allow stream download of album ZIPs

Browse Source 
extend ServeStatic handler to allow programatically calling the handle()
function from within in-progress Requests

also use file's timestamp as file's modified time in the ZIP archive
This commit is contained in:
Bobby 2022-10-04 07:06:37 +07:00
parent 97ffa67975
commit 93621afe94
No known key found for this signature in database
GPG Key ID: 941839794CBF5A09
2 changed files with 91 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
controllers/albumsController.js
View File

@ -1,4 +1,6 @@
const contentDisposition = require('content-disposition')
const EventEmitter = require('events')
const fsPromises = require('fs/promises')
const jetpack = require('fs-jetpack')
const path = require('path')
const randomstring = require('randomstring')
@ -6,6 +8,7 @@ const Zip = require('jszip')
const paths = require('./pathsController')
const perms = require('./permissionController')
const utils = require('./utilsController')
const ServeStatic = require('./handlers/ServeStatic')
const ClientError = require('./utils/ClientError')
const ServerError = require('./utils/ServerError')
const config = require('./../config')
@ -30,7 +33,7 @@ const albumsPerPage = config.dashboard
const zipMaxTotalSize = parseInt(config.cloudflare.zipMaxTotalSize)
const zipMaxTotalSizeBytes = zipMaxTotalSize * 1e6
const zipOptions = config.uploads.jsZipOptions
const zipOptions = config.uploads.jsZipOptions || {}
// Force 'type' option to 'nodebuffer'
zipOptions.type = 'nodebuffer'
@ -38,8 +41,7 @@ zipOptions.type = 'nodebuffer'
// Apply fallbacks for missing config values
if (zipOptions.streamFiles === undefined) zipOptions.streamFiles = true
if (zipOptions.compression === undefined) zipOptions.compression = 'DEFLATE'
if (zipOptions.compressionOptions === undefined) zipOptions.compressionOptions = {}
if (zipOptions.compressionOptions.level === undefined) zipOptions.compressionOptions.level = 1
if (zipOptions.compressionOptions === undefined) zipOptions.compressionOptions = { level: 1 }
self.zipEmitters = new Map()
@ -51,6 +53,9 @@ class ZipEmitter extends EventEmitter {
}
}
// ServeStatic instance to handle downloading of album ZIP archives
const serveAlbumZipInstance = new ServeStatic(paths.zips)
self.getUniqueAlbumIdentifier = async res => {
for (let i = 0; i < utils.idMaxTries; i++) {
const identifier = randomstring.generate(config.uploads.albumIdentifierLength)
@ -575,42 +580,46 @@ self.generateZip = async (req, res) => {
return res.redirect(`${album.identifier}?v=${album.editedAt}`)
}
// Downloading existing album ZIP archive if still valid
if (album.zipGeneratedAt > album.editedAt) {
const filePath = path.join(paths.zips, `${identifier}.zip`)
const stat = await jetpack.inspectAsync(filePath)
if (stat.type === 'file') {
const readStream = jetpack.createReadStream(filePath)
readStream.on('error', error => {
readStream.destroy()
logger.error(error)
try {
const filePath = path.join(paths.zips, `${identifier}.zip`)
const stat = await fsPromises.stat(filePath)
return serveAlbumZipInstance.handle(req, res, filePath, stat, (req, res) => {
res.header('Content-Disposition', contentDisposition(`${album.name}.zip`, { type: 'inline' }))
})
// 2nd param will be set as Content-Length header (must be number)
await res.stream(readStream, stat.size)
return
} catch (error) {
// Re-throw non-ENOENT error
if (error.code !== 'ENOENT') {
throw error
}
}
}
// If EventEmitter already exists for this album ZIP generation, wait for it
if (self.zipEmitters.has(identifier)) {
return new Promise((resolve, reject) => {
logger.log(`Waiting previous zip task for album: ${identifier}.`)
self.zipEmitters.get(identifier).once('done', (filePath, fileName, clientErr) => {
if (filePath && fileName) {
resolve({ filePath, fileName })
} else if (clientErr) {
reject(clientErr)
self.zipEmitters.get(identifier).once('done', (result, clientErr) => {
if (clientErr || !result) {
return reject(clientErr || new ServerError())
}
return resolve(result)
})
}).then(obj => {
return res.download(obj.filePath, obj.fileName)
})
}).then(async result =>
serveAlbumZipInstance.handle(req, res, result.path, result.stat, (req, res) => {
res.header('Content-Disposition', contentDisposition(result.name, { type: 'inline' }))
})
)
}
// Create EventEmitter for this album ZIP generation
self.zipEmitters.set(identifier, new ZipEmitter(identifier))
logger.log(`Starting zip task for album: ${identifier}.`)
const files = await utils.db.table('files')
.select('name', 'size')
.select('name', 'size', 'timestamp')
.where('albumid', album.id)
if (files.length === 0) {
logger.log(`Finished zip task for album: ${identifier} (no files).`)
@ -635,12 +644,14 @@ self.generateZip = async (req, res) => {
const archive = new Zip()
try {
// Since we are adding all files concurrently,
// their order in the ZIP file may not be in alphabetical order.
// However, ZIP viewers in general should sort the files themselves.
for (const file of files) {
const fullPath = path.join(paths.uploads, file.name)
archive.file(file.name, jetpack.createReadStream(fullPath))
archive.file(file.name, jetpack.createReadStream(fullPath), {
// Use file's upload timestamp as file's modified time in the ZIP archive.
// Timezone information does not seem to persist,
// so the displayed modified time will likely always be in UTC+0.
date: new Date(file.timestamp * 1000)
})
}
await new Promise((resolve, reject) => {
archive.generateNodeStream(zipOptions)
@ -660,11 +671,19 @@ self.generateZip = async (req, res) => {
.update('zipGeneratedAt', Math.floor(Date.now() / 1000))
utils.invalidateStatsCache('albums')
const filePath = path.join(paths.zips, `${identifier}.zip`)
const fileName = `${album.name}.zip`
const result = {
path: path.join(paths.zips, `${identifier}.zip`),
name: `${album.name}.zip`
}
result.stat = await fsPromises.stat(result.path)
self.zipEmitters.get(identifier).emit('done', filePath, fileName)
return res.download(filePath, fileName)
// Notify all other awaiting Requests, if any
self.zipEmitters.get(identifier).emit('done', result)
// Conclude this Request by streaming the album ZIP archive
return serveAlbumZipInstance.handle(req, res, result.path, result.stat, (req, res) => {
res.header('Content-Disposition', contentDisposition(result.name, { type: 'inline' }))
})
}
self.addFiles = async (req, res) => {

68
controllers/handlers/ServeStatic.js
View File

@ -105,37 +105,20 @@ class ServeStatic {
this.#options = options
}
async #get (fullPath) {
const stat = await fsPromises.stat(fullPath)
if (stat.isDirectory()) return
return stat
}
async #handler (req, res) {
if (this.#options.ignorePatterns && this.#options.ignorePatterns.some(pattern => req.path.startsWith(pattern))) {
return errors.handleNotFound(req, res)
}
const fullPath = this.directory + req.path
const stat = await this.#get(fullPath)
.catch(error => {
// Only re-throw errors if not due to missing files
if (error.code !== 'ENOENT') {
throw error
}
})
if (stat === undefined) {
return errors.handleNotFound(req, res)
}
// Can be programatically called from within in-progress Requests
// Essentially stream-based alternative for Response.download() or Response.send()
async #handle (req, res, fullPath, stat, setHeaders) {
// Set Content-Type
res.type(req.path)
res.type(fullPath)
// Set header fields
await this.#setHeaders(req, res, stat)
// Per-request setHeaders, if required
if (typeof setHeaders === 'function') {
setHeaders(req, res)
}
// Conditional GET support
if (serveUtils.assertConditionalGET(req, res)) {
return res.end()
@ -167,6 +150,35 @@ class ServeStatic {
return this.#stream(req, res, fullPath, result)
}
async #get (fullPath) {
const stat = await fsPromises.stat(fullPath)
if (stat.isDirectory()) return
return stat
}
// As route handler function for HyperExpress.any()
async #handler (req, res) {
if (this.#options.ignorePatterns && this.#options.ignorePatterns.some(pattern => req.path.startsWith(pattern))) {
return errors.handleNotFound(req, res)
}
const fullPath = this.directory + req.path
const stat = await this.#get(fullPath)
.catch(error => {
// Only re-throw errors if not due to missing files
if (error.code !== 'ENOENT') {
throw error
}
})
if (stat === undefined) {
return errors.handleNotFound(req, res)
}
return this.#handle(req, res, fullPath, stat)
}
async #setContentDisposition (req, res) {
// Do only if accessing files from uploads' root directory (i.e. not thumbs, etc.)
if (req.path.indexOf('/', 1) !== -1) return
@ -248,6 +260,10 @@ class ServeStatic {
return res.stream(readStream, result.length)
}
get handle () {
return this.#handle.bind(this)
}
get handler () {
return this.#handler.bind(this)
}