mirror of
https://github.com/BobbyWibowo/lolisafe.git
synced 2025-01-31 07:11:33 +00:00
Implemented parallel URL uploads
This doesn't use the server's built-in ability to accept multiple URLs per API request. It behaves the same as regular uploads, in that it executes one API call per file, simultaneously. I figured this is a better implementation to shift queues faster. --- Fetch error from URL uploads due to exceeding size limit will no longer be logged in server's console. Clients will also see better formatted error message for URL uploads' file size limit errors. --- Bumped dependencies: knex: 0.20.2 -> 0.20.3 systeminformation: 4.15.3 -> 4.16.0 Bumped v1 version string
This commit is contained in:
Bobby Wibowo
parent
df1e835272
commit
337a0a61ff
2
TODO.md
2
TODO.md
@ -35,7 +35,7 @@ Consider remembering last pages of each individual albums as well. When deleting
|
||||
|
||||
Low priority:
|
||||
|
||||
* [ ] Parallel URL uploads.
|
||||
* [x] Parallel URL uploads.
|
||||
* [x] Delete user feature.
|
||||
* [ ] Bulk delete user feature.
|
||||
* [ ] Bulk disable user feature.
|
||||
|
||||
@ -363,8 +363,14 @@ self.actuallyUploadUrls = async (req, res, user, albumid, age) => {
|
||||
utils.unlinkFile(file).catch(logger.error)
|
||||
))
|
||||
|
||||
// Re-throw error
|
||||
throw error
|
||||
const errorString = error.toString()
|
||||
const suppress = [
|
||||
/ over limit:/
|
||||
]
|
||||
if (!suppress.some(t => t.test(errorString)))
|
||||
throw error
|
||||
else
|
||||
throw errorString
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
2
dist/js/home.js
vendored
2
dist/js/home.js
vendored
File diff suppressed because one or more lines are too long
2
dist/js/home.js.map
vendored
2
dist/js/home.js.map
vendored
File diff suppressed because one or more lines are too long
@ -37,7 +37,7 @@
|
||||
"fluent-ffmpeg": "^2.1.2",
|
||||
"helmet": "^3.21.2",
|
||||
"jszip": "^3.2.2",
|
||||
"knex": "^0.20.2",
|
||||
"knex": "^0.20.3",
|
||||
"multer": "^1.4.2",
|
||||
"node-fetch": "^2.6.0",
|
||||
"nunjucks": "^3.2.0",
|
||||
@ -45,7 +45,7 @@
|
||||
"readline": "^1.3.0",
|
||||
"sharp": "^0.23.3",
|
||||
"sqlite3": "^4.1.0",
|
||||
"systeminformation": "^4.15.3"
|
||||
"systeminformation": "^4.16.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"browserslist": "^4.7.3",
|
||||
|
||||
172
src/js/home.js
172
src/js/home.js
@ -43,6 +43,10 @@ const page = {
|
||||
clipboardJS: null,
|
||||
lazyLoad: null,
|
||||
|
||||
// additional vars for url uploads
|
||||
urlsQueue: [],
|
||||
activeUrlsQueue: 0,
|
||||
|
||||
// Include BMP for uploads preview only, cause the real images will be used
|
||||
// Sharp isn't capable of making their thumbnails for dashboard and album public pages
|
||||
imageExts: ['.webp', '.jpg', '.jpeg', '.bmp', '.gif', '.png', '.tiff', '.tif', '.svg'],
|
||||
@ -215,7 +219,7 @@ page.prepareUpload = () => {
|
||||
page.urlMaxSizeBytes = page.urlMaxSize * 1e6
|
||||
urlMaxSize.innerHTML = page.getPrettyBytes(page.urlMaxSizeBytes)
|
||||
document.querySelector('#uploadUrls').addEventListener('click', event => {
|
||||
page.uploadUrls(event.currentTarget)
|
||||
page.addUrlsToQueue()
|
||||
})
|
||||
}
|
||||
|
||||
@ -382,17 +386,17 @@ page.prepareDropzone = () => {
|
||||
`${prefix} ${percentage}%${prettyBytesPerSec ? ` at ~${prettyBytesPerSec}/s` : ''}`
|
||||
})
|
||||
|
||||
this.on('success', (file, response) => {
|
||||
if (!response) return
|
||||
this.on('success', (file, data) => {
|
||||
if (!data) return
|
||||
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
|
||||
if (response.success === false) {
|
||||
file.previewElement.querySelector('.error').innerHTML = response.description
|
||||
if (data.success === false) {
|
||||
file.previewElement.querySelector('.error').innerHTML = data.description
|
||||
file.previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
}
|
||||
|
||||
if (response.files && response.files[0])
|
||||
page.updateTemplate(file, response.files[0])
|
||||
if (Array.isArray(data.files) && data.files[0])
|
||||
page.updateTemplate(file, data.files[0])
|
||||
})
|
||||
|
||||
this.on('error', (file, error) => {
|
||||
@ -404,7 +408,6 @@ page.prepareDropzone = () => {
|
||||
page.updateTemplateIcon(file.previewElement, 'icon-block')
|
||||
|
||||
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
file.previewElement.querySelector('.name').innerHTML = file.name
|
||||
|
||||
file.previewElement.querySelector('.error').innerHTML = error.description || error
|
||||
file.previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
@ -454,85 +457,96 @@ page.prepareDropzone = () => {
|
||||
})
|
||||
}
|
||||
|
||||
page.uploadUrls = button => {
|
||||
const tabDiv = document.querySelector('#tab-urls')
|
||||
if (!tabDiv || button.classList.contains('is-loading'))
|
||||
return
|
||||
|
||||
button.classList.add('is-loading')
|
||||
|
||||
function done (error) {
|
||||
if (error) swal('An error occurred!', error, 'error')
|
||||
button.classList.remove('is-loading')
|
||||
}
|
||||
|
||||
function run () {
|
||||
const headers = {
|
||||
token: page.token,
|
||||
albumid: page.album,
|
||||
age: page.uploadAge,
|
||||
filelength: page.fileLength
|
||||
}
|
||||
|
||||
const previewsContainer = tabDiv.querySelector('.uploads')
|
||||
|
||||
const urls = document.querySelector('#urls').value
|
||||
.split(/\r?\n/)
|
||||
.filter(url => {
|
||||
return url.trim().length
|
||||
})
|
||||
document.querySelector('#urls').value = urls.join('\n')
|
||||
|
||||
if (!urls.length)
|
||||
return done('You have not entered any URLs.')
|
||||
|
||||
tabDiv.querySelector('.uploads').classList.remove('is-hidden')
|
||||
|
||||
const files = urls.map(url => {
|
||||
const previewTemplate = document.createElement('template')
|
||||
previewTemplate.innerHTML = page.previewTemplate.trim()
|
||||
|
||||
const previewElement = previewTemplate.content.firstChild
|
||||
previewElement.querySelector('.name').innerHTML = url
|
||||
previewElement.querySelector('.descriptive-progress').innerHTML = 'Waiting in queue\u2026'
|
||||
|
||||
previewsContainer.appendChild(previewElement)
|
||||
return { url, previewElement }
|
||||
page.addUrlsToQueue = () => {
|
||||
const urls = document.querySelector('#urls').value
|
||||
.split(/\r?\n/)
|
||||
.filter(url => {
|
||||
return url.trim().length
|
||||
})
|
||||
|
||||
function post (i) {
|
||||
if (i === files.length)
|
||||
return done()
|
||||
if (!urls.length)
|
||||
return swal('An error occurred!', 'You have not entered any URLs.', 'error')
|
||||
|
||||
function posted (result) {
|
||||
files[i].previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
const tabDiv = document.querySelector('#tab-urls')
|
||||
tabDiv.querySelector('.uploads').classList.remove('is-hidden')
|
||||
|
||||
if (result.success) {
|
||||
page.updateTemplate(files[i], result.files[0])
|
||||
} else {
|
||||
page.updateTemplateIcon(files[i].previewElement, 'icon-block')
|
||||
files[i].previewElement.querySelector('.error').innerHTML = result.description
|
||||
files[i].previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
}
|
||||
for (let i = 0; i < urls.length; i++) {
|
||||
const previewTemplate = document.createElement('template')
|
||||
previewTemplate.innerHTML = page.previewTemplate.trim()
|
||||
|
||||
return post(i + 1)
|
||||
}
|
||||
const previewElement = previewTemplate.content.firstChild
|
||||
previewElement.querySelector('.name').innerHTML = urls[i]
|
||||
previewElement.querySelector('.descriptive-progress').innerHTML = 'Waiting in queue\u2026'
|
||||
|
||||
files[i].previewElement.querySelector('.descriptive-progress').innerHTML =
|
||||
'Waiting for server to fetch URL\u2026'
|
||||
const previewsContainer = tabDiv.querySelector('.uploads')
|
||||
previewsContainer.appendChild(previewElement)
|
||||
|
||||
return axios.post('api/upload', { urls: [files[i].url] }, { headers }).then(response => {
|
||||
return posted(response.data)
|
||||
}).catch(error => {
|
||||
return posted({
|
||||
success: false,
|
||||
description: error.response ? error.response.data.description : error.toString()
|
||||
})
|
||||
})
|
||||
}
|
||||
return post(0)
|
||||
page.urlsQueue.push({
|
||||
url: urls[i],
|
||||
previewElement
|
||||
})
|
||||
}
|
||||
return run()
|
||||
|
||||
page.processUrlsQueue()
|
||||
document.querySelector('#urls').value = ''
|
||||
}
|
||||
|
||||
page.processUrlsQueue = () => {
|
||||
if (!page.urlsQueue.length) return
|
||||
|
||||
function finishedUrlUpload (file, data) {
|
||||
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
|
||||
if (data.success === false) {
|
||||
const match = data.description.match(/ over limit: (\d+)$/)
|
||||
if (match && match[1])
|
||||
data.description = `File exceeded limit of ${page.getPrettyBytes(match[1])}.`
|
||||
|
||||
file.previewElement.querySelector('.error').innerHTML = data.description
|
||||
file.previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
}
|
||||
|
||||
if (Array.isArray(data.files) && data.files[0])
|
||||
page.updateTemplate(file, data.files[0])
|
||||
|
||||
page.activeUrlsQueue--
|
||||
return shiftQueue()
|
||||
}
|
||||
|
||||
function initUrlUpload (file) {
|
||||
file.previewElement.querySelector('.descriptive-progress').innerHTML =
|
||||
'Waiting for server to fetch URL\u2026'
|
||||
|
||||
return axios.post('api/upload', {
|
||||
urls: [file.url]
|
||||
}, {
|
||||
headers: {
|
||||
token: page.token,
|
||||
albumid: page.album,
|
||||
age: page.uploadAge,
|
||||
filelength: page.fileLength
|
||||
}
|
||||
}).catch(error => {
|
||||
// Format error for display purpose
|
||||
return error.response.data ? error.response : {
|
||||
data: {
|
||||
success: false,
|
||||
description: error.toString()
|
||||
}
|
||||
}
|
||||
}).then(response => {
|
||||
return finishedUrlUpload(file, response.data)
|
||||
})
|
||||
}
|
||||
|
||||
function shiftQueue () {
|
||||
while (page.urlsQueue.length && (page.activeUrlsQueue < page.parallelUploads)) {
|
||||
page.activeUrlsQueue++
|
||||
initUrlUpload(page.urlsQueue.shift())
|
||||
}
|
||||
}
|
||||
|
||||
return shiftQueue()
|
||||
}
|
||||
|
||||
page.updateTemplateIcon = (templateElement, iconClass) => {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
{
|
||||
"1": "1574791009",
|
||||
"1": "1575024012",
|
||||
"2": "1568894058",
|
||||
"3": "1568894058",
|
||||
"4": "1568894058",
|
||||
|
||||
52
yarn.lock
52
yarn.lock
@ -203,9 +203,9 @@
|
||||
integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
|
||||
|
||||
"@types/node@*":
|
||||
version "12.12.12"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.12.tgz#529bc3e73dbb35dd9e90b0a1c83606a9d3264bdb"
|
||||
integrity sha512-MGuvYJrPU0HUwqF7LqvIj50RZUX23Z+m583KBygKYUZLlZ88n6w28XRNJRJgsHukLEnLz6w6SvxZoLgbr5wLqQ==
|
||||
version "12.12.14"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.14.tgz#1c1d6e3c75dba466e0326948d56e8bd72a1903d2"
|
||||
integrity sha512-u/SJDyXwuihpwjXy7hOOghagLEV1KdAST6syfnOk6QZAMzZuWZqXy5aYYZbh8Jdpd4escVFP0MvftHNDb9pruA==
|
||||
|
||||
"@types/q@^1.5.1":
|
||||
version "1.5.2"
|
||||
@ -646,9 +646,9 @@ aws-sign2@~0.7.0:
|
||||
integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=
|
||||
|
||||
aws4@^1.8.0:
|
||||
version "1.8.0"
|
||||
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
|
||||
integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
|
||||
version "1.9.0"
|
||||
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.9.0.tgz#24390e6ad61386b0a747265754d2a17219de862c"
|
||||
integrity sha512-Uvq6hVe90D0B2WEnUqtdgY1bATGz3mw33nH9Y+dmA+w5DHvUmBgkr5rM/KCHpCsiFNRUfokW/szpPPgMK2hm4A==
|
||||
|
||||
bach@^1.0.0:
|
||||
version "1.2.0"
|
||||
@ -721,9 +721,9 @@ bl@^3.0.0:
|
||||
readable-stream "^3.0.1"
|
||||
|
||||
bluebird@^3.7.1:
|
||||
version "3.7.1"
|
||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.1.tgz#df70e302b471d7473489acf26a93d63b53f874de"
|
||||
integrity sha512-DdmyoGCleJnkbp3nkbxTLJ18rjDsE4yCggEwKNXkeV123sPNfOCYeDoeuOY+F2FrSjO1YXcTU+dsy96KMy+gcg==
|
||||
version "3.7.2"
|
||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
|
||||
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
|
||||
|
||||
body-parser@1.19.0, body-parser@^1.19.0:
|
||||
version "1.19.0"
|
||||
@ -2216,9 +2216,9 @@ express@^4.17.1:
|
||||
vary "~1.1.2"
|
||||
|
||||
ext@^1.1.2:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/ext/-/ext-1.2.0.tgz#8dd8d2dd21bcced3045be09621fa0cbf73908ba4"
|
||||
integrity sha512-0ccUQK/9e3NreLFg6K6np8aPyRgwycx+oFGtfx1dSp7Wj00Ozw9r05FgBRlzjf2XBM7LAzwgLyDscRrtSU91hA==
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/ext/-/ext-1.3.0.tgz#21526eb296753fed34b620d4a69e3911065fa925"
|
||||
integrity sha512-LErT9cIGZZjSvFkyocVXXeYlj7z8xiA+4oQlM9cX4X/Kfc18cefv3Dd9mNKwFuzUJ7neMMAQz1u1r3gBa/6wGg==
|
||||
dependencies:
|
||||
type "^2.0.0"
|
||||
|
||||
@ -3826,10 +3826,10 @@ kind-of@^6.0.0, kind-of@^6.0.2:
|
||||
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051"
|
||||
integrity sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==
|
||||
|
||||
knex@^0.20.2:
|
||||
version "0.20.2"
|
||||
resolved "https://registry.yarnpkg.com/knex/-/knex-0.20.2.tgz#7429577a95a10f4a4e3090c23b559fed20343b4a"
|
||||
integrity sha512-nw7/RsaZrIGdzbsb1evcEaZv8sL/Ji2W7o5OoF0NIKei4ySU01D4G5mRNVNtneoLoPjUMgqSFRanabhGacJUIA==
|
||||
knex@^0.20.3:
|
||||
version "0.20.3"
|
||||
resolved "https://registry.yarnpkg.com/knex/-/knex-0.20.3.tgz#85178cd6873f75827be86d054c4e117bb4d9657b"
|
||||
integrity sha512-zzYO34pSCCYVqRTbCp8xL+Z7fvHQl5anif3Oacu6JaHFDubB7mFGWRRJBNSO3N8Ql4g4CxUgBctaPiliwoOsNA==
|
||||
dependencies:
|
||||
bluebird "^3.7.1"
|
||||
colorette "1.1.0"
|
||||
@ -4414,9 +4414,9 @@ nocache@2.1.0:
|
||||
integrity sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==
|
||||
|
||||
node-abi@^2.7.0:
|
||||
version "2.12.0"
|
||||
resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.12.0.tgz#40e9cfabdda1837863fa825e7dfa0b15686adf6f"
|
||||
integrity sha512-VhPBXCIcvmo/5K8HPmnWJyyhvgKxnHTUMXR/XwGHV68+wrgkzST4UmQrY/XszSWA5dtnXpNp528zkcyJ/pzVcw==
|
||||
version "2.13.0"
|
||||
resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.13.0.tgz#e2f2ec444d0aca3ea1b3874b6de41d1665828f63"
|
||||
integrity sha512-9HrZGFVTR5SOu3PZAnAY2hLO36aW1wmA+FDsVkr85BTST32TLCA1H/AEcatVRAsWLyXS3bqUDYCAjq5/QGuSTA==
|
||||
dependencies:
|
||||
semver "^5.4.1"
|
||||
|
||||
@ -6179,9 +6179,9 @@ resolve-url@^0.2.1:
|
||||
integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=
|
||||
|
||||
resolve@^1.1.6, resolve@^1.1.7, resolve@^1.10.0, resolve@^1.10.1, resolve@^1.11.0, resolve@^1.3.2, resolve@^1.4.0, resolve@^1.5.0:
|
||||
version "1.12.2"
|
||||
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.12.2.tgz#08b12496d9aa8659c75f534a8f05f0d892fff594"
|
||||
integrity sha512-cAVTI2VLHWYsGOirfeYVVQ7ZDejtQ9fp4YhYckWDEkFfqbVjaT11iM8k6xSAfGFMM+gDpZjMnFssPu8we+mqFw==
|
||||
version "1.13.1"
|
||||
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.13.1.tgz#be0aa4c06acd53083505abb35f4d66932ab35d16"
|
||||
integrity sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==
|
||||
dependencies:
|
||||
path-parse "^1.0.6"
|
||||
|
||||
@ -6892,10 +6892,10 @@ svgo@^1.0.0:
|
||||
unquote "~1.1.1"
|
||||
util.promisify "~1.0.0"
|
||||
|
||||
systeminformation@^4.15.3:
|
||||
version "4.15.3"
|
||||
resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-4.15.3.tgz#639cdc224b5c3811f1a0bfba33f869bbc6fb930f"
|
||||
integrity sha512-Fx2ARGHtLl2/xLeNoTR8/doXSxUXuAzIN+dyCK9O43j/UETLBt77yTEbTxmYsVD47PYjX1iQTdcY41CZckY+zg==
|
||||
systeminformation@^4.16.0:
|
||||
version "4.16.0"
|
||||
resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-4.16.0.tgz#d92ce821efdab720496c60656bf65cc68fb03f8c"
|
||||
integrity sha512-1FjxPJSw7ad0zug+1YIQATj6Cn+wM5OBASEpjohEeOD2EGPIf0Cnhthd1L2O1YX+wKgOMuPldGfxYdo8yNHEIg==
|
||||
|
||||
table@^5.2.3:
|
||||
version "5.4.6"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user