mirror of
https://github.com/BobbyWibowo/lolisafe.git
synced 2025-01-18 17:21:33 +00:00
cleaned up some scripts
This commit is contained in:
Bobby Wibowo
parent
3de44243a2
commit
7f3b947eb0
@ -5,19 +5,17 @@ const config = require('./../config')
|
||||
const db = require('knex')(config.database)
|
||||
|
||||
const self = {
|
||||
mode: null
|
||||
}
|
||||
|
||||
self.getFiles = async directory => {
|
||||
const names = await paths.readdir(directory)
|
||||
const files = []
|
||||
for (const name of names) {
|
||||
const lstat = await paths.lstat(path.join(directory, name))
|
||||
if (lstat.isFile() && !name.startsWith('.')) {
|
||||
files.push(name)
|
||||
getFiles: async directory => {
|
||||
const names = await paths.readdir(directory)
|
||||
const files = []
|
||||
for (const name of names) {
|
||||
const lstat = await paths.lstat(path.join(directory, name))
|
||||
if (lstat.isFile() && !name.startsWith('.')) {
|
||||
files.push(name)
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
;(async () => {
|
||||
@ -37,8 +35,10 @@ self.getFiles = async directory => {
|
||||
`).trim())
|
||||
}
|
||||
|
||||
self.mode = parseInt(args[0]) || 0
|
||||
const dryrun = self.mode === 0
|
||||
const mode = parseInt(args[0]) || 0
|
||||
const dryrun = mode === 0
|
||||
|
||||
console.log('Querying uploads\u2026')
|
||||
|
||||
const uploads = await self.getFiles(paths.uploads)
|
||||
console.log(`Uploads: ${uploads.length}`)
|
||||
@ -49,18 +49,19 @@ self.getFiles = async directory => {
|
||||
console.log(`- In DB: ${uploadsDb.length}`)
|
||||
|
||||
const uploadsNotInDb = uploads.filter(upload => !uploadsDb.includes(upload))
|
||||
console.log(`- Not in DB: ${uploadsNotInDb.length}`)
|
||||
console.log(`- Stray: ${uploadsNotInDb.length}`)
|
||||
|
||||
const thumbs = await self.getFiles(paths.thumbs)
|
||||
console.log(`Thumbs: ${thumbs.length}`)
|
||||
console.log(`Thumbs : ${thumbs.length}`)
|
||||
|
||||
const uploadsDbSet = new Set(uploadsDb.map(upload => upload.split('.')[0]))
|
||||
const thumbsNotInDb = thumbs.filter(thumb => !uploadsDbSet.has(thumb.slice(0, -4)))
|
||||
console.log(`- Not in DB: ${thumbsNotInDb.length}`)
|
||||
console.log(`- Stray: ${thumbsNotInDb.length}`)
|
||||
|
||||
if (dryrun) {
|
||||
console.log('U:', uploadsNotInDb.join(', '))
|
||||
console.log('T:', thumbsNotInDb.join(', '))
|
||||
console.log('Stray uploads:', uploadsNotInDb.join(', '))
|
||||
console.log('Stray thumbs :', thumbsNotInDb.join(', '))
|
||||
console.log('INFO: This was a dry run. No files had been deleted.')
|
||||
} else if (!dryrun) {
|
||||
for (const upload of uploadsNotInDb) {
|
||||
await paths.unlink(path.join(paths.uploads, upload))
|
||||
|
||||
@ -1,9 +1,5 @@
|
||||
const utils = require('../controllers/utilsController')
|
||||
|
||||
const self = {
|
||||
mode: null
|
||||
}
|
||||
|
||||
;(async () => {
|
||||
const location = process.argv[1].replace(process.cwd() + '/', '')
|
||||
const args = process.argv.slice(2)
|
||||
@ -22,9 +18,9 @@ const self = {
|
||||
`).trim())
|
||||
}
|
||||
|
||||
self.mode = parseInt(args[0]) || 0
|
||||
const dryrun = self.mode === 0
|
||||
const quiet = self.mode === 2
|
||||
const mode = parseInt(args[0]) || 0
|
||||
const dryrun = mode === 0
|
||||
const quiet = mode === 2
|
||||
|
||||
const result = await utils.bulkDeleteExpired(dryrun, true)
|
||||
|
||||
|
||||
@ -6,44 +6,41 @@ const utils = require('../controllers/utilsController')
|
||||
const config = require('./../config')
|
||||
const db = require('knex')(config.database)
|
||||
|
||||
const self = {
|
||||
mode: null
|
||||
}
|
||||
|
||||
;(async () => {
|
||||
const location = process.argv[1].replace(process.cwd() + '/', '')
|
||||
const args = process.argv.slice(2)
|
||||
|
||||
self.mode = parseInt(args[0])
|
||||
const mode = parseInt(args[0])
|
||||
|
||||
if (![0, 1, 2].includes(self.mode) ||
|
||||
if (![0, 1, 2].includes(mode) ||
|
||||
args.includes('--help') ||
|
||||
args.includes('-h')) {
|
||||
return console.log(utils.stripIndents(`
|
||||
Rebuild file hashes.
|
||||
|
||||
Usage :
|
||||
Usage:
|
||||
node ${location} <mode=0|1|2> [parallel]
|
||||
|
||||
mode:
|
||||
0 = Dry run (recalculate hashes, print them, but do NOT store to DB).
|
||||
1 = Recalculate hashes, print them, and store to DB.
|
||||
2 = Quiet (recalculate hashes and store to DB).
|
||||
1 = Recalculate hashes and store to DB.
|
||||
2 = Verbose (recalculate hashes, print them, and store to DB).
|
||||
|
||||
parallel:
|
||||
Amount of uploads to hash in parallel (not to be confused with multi-threading).
|
||||
`).trim())
|
||||
}
|
||||
|
||||
const dryrun = self.mode === 0
|
||||
const quiet = self.mode === 2
|
||||
const parallel = Math.min(Math.max(parseInt(args[1]), 0), 32) || 8
|
||||
const dryrun = mode === 0
|
||||
const verbose = [0, 2].includes(mode)
|
||||
const parallel = Math.max(parseInt(args[1]), 1) || 1
|
||||
|
||||
console.log(`Parallel: ${parallel}`)
|
||||
console.log('Querying uploads\u2026')
|
||||
const hrstart = process.hrtime()
|
||||
const uploads = await db.table('files')
|
||||
.select('id', 'name', 'hash')
|
||||
console.log(`Uploads : ${uploads.length}`)
|
||||
console.log(`Parallel: ${parallel}`)
|
||||
|
||||
let lastProgressOut
|
||||
await utils.parallelLimit(uploads.map(upload => {
|
||||
@ -53,7 +50,7 @@ const self = {
|
||||
.pipe(blake3.createHash())
|
||||
.on('data', async hasher => {
|
||||
const hash = hasher.toString('hex')
|
||||
if (!quiet) console.log(`${upload.name}: ${hash}`)
|
||||
if (verbose) console.log(`${upload.name}: ${hash}`)
|
||||
if (!dryrun && upload.hash !== hash) {
|
||||
await db.table('files')
|
||||
.update('hash', hash)
|
||||
|
||||
@ -1,29 +1,26 @@
|
||||
const path = require('path')
|
||||
const paths = require('../controllers/pathsController')
|
||||
const utils = require('../controllers/utilsController')
|
||||
const config = require('./../config')
|
||||
const db = require('knex')(config.database)
|
||||
|
||||
const self = {
|
||||
mode: null,
|
||||
force: null,
|
||||
verbose: null,
|
||||
cfcache: null
|
||||
}
|
||||
|
||||
self.mayGenerateThumb = extname => {
|
||||
return ([1, 3].includes(self.mode) && utils.imageExts.includes(extname)) ||
|
||||
mayGenerateThumb: extname => {
|
||||
return ([1, 3].includes(self.mode) && utils.imageExts.includes(extname)) ||
|
||||
([2, 3].includes(self.mode) && utils.videoExts.includes(extname))
|
||||
}
|
||||
|
||||
self.getFiles = async directory => {
|
||||
const names = await paths.readdir(directory)
|
||||
const files = []
|
||||
for (const name of names) {
|
||||
const lstat = await paths.lstat(path.join(directory, name))
|
||||
if (lstat.isFile() && !name.startsWith('.')) {
|
||||
files.push(name)
|
||||
},
|
||||
getFiles: async directory => {
|
||||
const names = await paths.readdir(directory)
|
||||
const files = []
|
||||
for (const name of names) {
|
||||
const lstat = await paths.lstat(path.join(directory, name))
|
||||
if (lstat.isFile() && !name.startsWith('.')) {
|
||||
files.push(name)
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
;(async () => {
|
||||
@ -31,69 +28,92 @@ self.getFiles = async directory => {
|
||||
const args = process.argv.slice(2)
|
||||
|
||||
self.mode = parseInt(args[0])
|
||||
self.force = parseInt(args[1]) || 0
|
||||
self.verbose = parseInt(args[2]) || 0
|
||||
self.cfcache = parseInt(args[3]) || 0
|
||||
const force = parseInt(args[1]) || 0
|
||||
const verbose = parseInt(args[2]) || 0
|
||||
const cfcache = parseInt(args[3]) || 0
|
||||
const parallel = Math.max(parseInt(args[1]), 1) || 1
|
||||
|
||||
if (![1, 2, 3].includes(self.mode) ||
|
||||
![0, 1].includes(self.force) ||
|
||||
![0, 1].includes(self.verbose) ||
|
||||
![0, 1].includes(force) ||
|
||||
![0, 1, 2].includes(verbose) ||
|
||||
![0, 1].includes(cfcache) ||
|
||||
args.includes('--help') ||
|
||||
args.includes('-h')) {
|
||||
return console.log(utils.stripIndents(`
|
||||
Generate thumbnails.
|
||||
|
||||
Usage :
|
||||
node ${location} <mode=1|2|3> [force=0|1] [verbose=0|1] [cfcache=0|1]
|
||||
Usage:
|
||||
node ${location} <mode=1|2|3> [force=0|1] [verbose=0|1] [cfcache=0|1] [parallel]
|
||||
|
||||
mode : 1 = images only, 2 = videos only, 3 = both images and videos
|
||||
force : 0 = no force (default), 1 = overwrite existing thumbnails
|
||||
verbose: 0 = only print missing thumbs (default), 1 = print all
|
||||
cfcache: 0 = do not clear cloudflare cache (default), 1 = clear cloudflare cache
|
||||
mode : 1 = images only, 2 = videos only, 3 = both images and videos
|
||||
force : 0 = no force (default), 1 = overwrite existing thumbnails
|
||||
verbose : 0 = only print missing thumbs (default), 1 = print all, 2 = print nothing
|
||||
cfcache : 0 = do not clear cloudflare cache (default), 1 = clear cloudflare cache
|
||||
parallel: amount of thumbs to generate in parallel (not to be confused with multi-threading).
|
||||
`).trim())
|
||||
}
|
||||
|
||||
console.log(`Parallel: ${parallel}`)
|
||||
console.log('Looking through existing thumbnails\u2026')
|
||||
const uploads = await self.getFiles(paths.uploads)
|
||||
let thumbs = await self.getFiles(paths.thumbs)
|
||||
thumbs = thumbs.map(thumb => {
|
||||
const extname = path.extname(thumb)
|
||||
return thumb.slice(0, -extname.length)
|
||||
})
|
||||
const uploads = await db.table('files')
|
||||
.select('id', 'name')
|
||||
const thumbs = await self.getFiles(paths.thumbs)
|
||||
.then(thumbs => thumbs.map(thumb => {
|
||||
const extname = path.extname(thumb)
|
||||
return thumb.slice(0, -extname.length)
|
||||
}))
|
||||
|
||||
console.log(`Found ${thumbs.length} existing thumbnails (may include placeholder symlinks).`)
|
||||
if (!self.verbose) {
|
||||
console.log('Verbose logging disabled! Please be patient, this script may appear to be frozen but is actually working in the background.')
|
||||
}
|
||||
|
||||
const succeeded = []
|
||||
let error = 0
|
||||
let exists = 0
|
||||
let skipped = 0
|
||||
for (const upload of uploads) {
|
||||
const extname = utils.extname(upload)
|
||||
const basename = upload.slice(0, -extname.length)
|
||||
let lastProgressOut
|
||||
await utils.parallelLimit(uploads.map(async upload => {
|
||||
const extname = utils.extname(upload.name)
|
||||
const basename = upload.name.slice(0, -extname.length)
|
||||
|
||||
if (thumbs.includes(basename) && !self.force) {
|
||||
if (self.verbose) console.log(`${upload}: thumb exists.`)
|
||||
skipped++
|
||||
if (thumbs.includes(basename) && !force) {
|
||||
if (verbose === 1) {
|
||||
console.log(`${upload.name}: ALREADY EXISTS.`)
|
||||
}
|
||||
exists++
|
||||
} else if (!self.mayGenerateThumb(extname)) {
|
||||
if (self.verbose) console.log(`${upload}: extension skipped.`)
|
||||
if (verbose === 1) {
|
||||
console.log(`${upload.name}: EXTENSION SKIPPED.`)
|
||||
}
|
||||
skipped++
|
||||
} else {
|
||||
const start = Date.now()
|
||||
const generated = await utils.generateThumbs(upload, extname, self.force)
|
||||
console.log(`${upload}: ${(Date.now() - start) / 1000}s: ${generated ? 'OK' : 'ERROR'}`)
|
||||
generated ? succeeded.push(upload) : error++
|
||||
const generated = await utils.generateThumbs(upload.name, extname, force)
|
||||
if (verbose !== 2) {
|
||||
console.log(`${upload.name}: ${(Date.now() - start) / 1000}s: ${generated ? 'OK' : 'ERROR'}`)
|
||||
}
|
||||
generated ? succeeded.push({ upload, extname }) : error++
|
||||
}
|
||||
}
|
||||
console.log(`Success: ${succeeded.length}\nError: ${error}\nSkipped: ${skipped}`)
|
||||
}), parallel, progress => {
|
||||
const now = Date.now()
|
||||
if (!lastProgressOut || (now - lastProgressOut >= 1000) || progress.done === progress.total) {
|
||||
console.log(`Progress: ${progress.done}/${progress.total}`)
|
||||
lastProgressOut = now
|
||||
}
|
||||
})
|
||||
|
||||
if (self.cfcache && succeeded.length) {
|
||||
console.log(utils.stripIndents(`
|
||||
---
|
||||
Success: ${succeeded.length}
|
||||
Error: ${error}
|
||||
Already exists: ${exists}
|
||||
Extension skipped: ${skipped}
|
||||
---
|
||||
`).trim())
|
||||
|
||||
if (cfcache && succeeded.length) {
|
||||
console.log('Purging Cloudflare\'s cache...')
|
||||
const results = await utils.purgeCloudflareCache(succeeded.map(name => {
|
||||
const extname = utils.extname(name)
|
||||
return `thumbs/${name.slice(0, -extname.length)}.png`
|
||||
}), true, false)
|
||||
const results = await utils.purgeCloudflareCache(succeeded.map(data =>
|
||||
`thumbs/${data.upload.name.slice(0, -data.extname.length)}.png`
|
||||
), true, false)
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
if (results[i].errors.length) {
|
||||
results[i].errors.forEach(error => console.error(`CF: ${error}`))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user