mirror of
https://github.com/BobbyWibowo/lolisafe.git
synced 2025-01-31 07:11:33 +00:00
Implemented parallel URL uploads
This doesn't use the server's built-in ability to accept multiple URLs per API request. It behaves the same as regular uploads, in that it executes one API call per file, simultaneously. I figured this is a better implementation to shift queues faster. --- Fetch error from URL uploads due to exceeding size limit will no longer be logged in server's console. Clients will also see better formatted error message for URL uploads' file size limit errors. --- Bumped dependencies: knex: 0.20.2 -> 0.20.3 systeminformation: 4.15.3 -> 4.16.0 Bumped v1 version string
This commit is contained in:
Bobby Wibowo
parent
df1e835272
commit
337a0a61ff
2
TODO.md
2
TODO.md
@ -35,7 +35,7 @@ Consider remembering last pages of each individual albums as well. When deleting
|
||||
|
||||
Low priority:
|
||||
|
||||
* [ ] Parallel URL uploads.
|
||||
* [x] Parallel URL uploads.
|
||||
* [x] Delete user feature.
|
||||
* [ ] Bulk delete user feature.
|
||||
* [ ] Bulk disable user feature.
|
||||
|
||||
@ -363,8 +363,14 @@ self.actuallyUploadUrls = async (req, res, user, albumid, age) => {
|
||||
utils.unlinkFile(file).catch(logger.error)
|
||||
))
|
||||
|
||||
// Re-throw error
|
||||
const errorString = error.toString()
|
||||
const suppress = [
|
||||
/ over limit:/
|
||||
]
|
||||
if (!suppress.some(t => t.test(errorString)))
|
||||
throw error
|
||||
else
|
||||
throw errorString
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
2
dist/js/home.js
vendored
2
dist/js/home.js
vendored
File diff suppressed because one or more lines are too long
2
dist/js/home.js.map
vendored
2
dist/js/home.js.map
vendored
File diff suppressed because one or more lines are too long
@ -37,7 +37,7 @@
|
||||
"fluent-ffmpeg": "^2.1.2",
|
||||
"helmet": "^3.21.2",
|
||||
"jszip": "^3.2.2",
|
||||
"knex": "^0.20.2",
|
||||
"knex": "^0.20.3",
|
||||
"multer": "^1.4.2",
|
||||
"node-fetch": "^2.6.0",
|
||||
"nunjucks": "^3.2.0",
|
||||
@ -45,7 +45,7 @@
|
||||
"readline": "^1.3.0",
|
||||
"sharp": "^0.23.3",
|
||||
"sqlite3": "^4.1.0",
|
||||
"systeminformation": "^4.15.3"
|
||||
"systeminformation": "^4.16.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"browserslist": "^4.7.3",
|
||||
|
||||
130
src/js/home.js
130
src/js/home.js
@ -43,6 +43,10 @@ const page = {
|
||||
clipboardJS: null,
|
||||
lazyLoad: null,
|
||||
|
||||
// additional vars for url uploads
|
||||
urlsQueue: [],
|
||||
activeUrlsQueue: 0,
|
||||
|
||||
// Include BMP for uploads preview only, cause the real images will be used
|
||||
// Sharp isn't capable of making their thumbnails for dashboard and album public pages
|
||||
imageExts: ['.webp', '.jpg', '.jpeg', '.bmp', '.gif', '.png', '.tiff', '.tif', '.svg'],
|
||||
@ -215,7 +219,7 @@ page.prepareUpload = () => {
|
||||
page.urlMaxSizeBytes = page.urlMaxSize * 1e6
|
||||
urlMaxSize.innerHTML = page.getPrettyBytes(page.urlMaxSizeBytes)
|
||||
document.querySelector('#uploadUrls').addEventListener('click', event => {
|
||||
page.uploadUrls(event.currentTarget)
|
||||
page.addUrlsToQueue()
|
||||
})
|
||||
}
|
||||
|
||||
@ -382,17 +386,17 @@ page.prepareDropzone = () => {
|
||||
`${prefix} ${percentage}%${prettyBytesPerSec ? ` at ~${prettyBytesPerSec}/s` : ''}`
|
||||
})
|
||||
|
||||
this.on('success', (file, response) => {
|
||||
if (!response) return
|
||||
this.on('success', (file, data) => {
|
||||
if (!data) return
|
||||
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
|
||||
if (response.success === false) {
|
||||
file.previewElement.querySelector('.error').innerHTML = response.description
|
||||
if (data.success === false) {
|
||||
file.previewElement.querySelector('.error').innerHTML = data.description
|
||||
file.previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
}
|
||||
|
||||
if (response.files && response.files[0])
|
||||
page.updateTemplate(file, response.files[0])
|
||||
if (Array.isArray(data.files) && data.files[0])
|
||||
page.updateTemplate(file, data.files[0])
|
||||
})
|
||||
|
||||
this.on('error', (file, error) => {
|
||||
@ -404,7 +408,6 @@ page.prepareDropzone = () => {
|
||||
page.updateTemplateIcon(file.previewElement, 'icon-block')
|
||||
|
||||
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
file.previewElement.querySelector('.name').innerHTML = file.name
|
||||
|
||||
file.previewElement.querySelector('.error').innerHTML = error.description || error
|
||||
file.previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
@ -454,85 +457,96 @@ page.prepareDropzone = () => {
|
||||
})
|
||||
}
|
||||
|
||||
page.uploadUrls = button => {
|
||||
const tabDiv = document.querySelector('#tab-urls')
|
||||
if (!tabDiv || button.classList.contains('is-loading'))
|
||||
return
|
||||
|
||||
button.classList.add('is-loading')
|
||||
|
||||
function done (error) {
|
||||
if (error) swal('An error occurred!', error, 'error')
|
||||
button.classList.remove('is-loading')
|
||||
}
|
||||
|
||||
function run () {
|
||||
const headers = {
|
||||
token: page.token,
|
||||
albumid: page.album,
|
||||
age: page.uploadAge,
|
||||
filelength: page.fileLength
|
||||
}
|
||||
|
||||
const previewsContainer = tabDiv.querySelector('.uploads')
|
||||
|
||||
page.addUrlsToQueue = () => {
|
||||
const urls = document.querySelector('#urls').value
|
||||
.split(/\r?\n/)
|
||||
.filter(url => {
|
||||
return url.trim().length
|
||||
})
|
||||
document.querySelector('#urls').value = urls.join('\n')
|
||||
|
||||
if (!urls.length)
|
||||
return done('You have not entered any URLs.')
|
||||
return swal('An error occurred!', 'You have not entered any URLs.', 'error')
|
||||
|
||||
const tabDiv = document.querySelector('#tab-urls')
|
||||
tabDiv.querySelector('.uploads').classList.remove('is-hidden')
|
||||
|
||||
const files = urls.map(url => {
|
||||
for (let i = 0; i < urls.length; i++) {
|
||||
const previewTemplate = document.createElement('template')
|
||||
previewTemplate.innerHTML = page.previewTemplate.trim()
|
||||
|
||||
const previewElement = previewTemplate.content.firstChild
|
||||
previewElement.querySelector('.name').innerHTML = url
|
||||
previewElement.querySelector('.name').innerHTML = urls[i]
|
||||
previewElement.querySelector('.descriptive-progress').innerHTML = 'Waiting in queue\u2026'
|
||||
|
||||
const previewsContainer = tabDiv.querySelector('.uploads')
|
||||
previewsContainer.appendChild(previewElement)
|
||||
return { url, previewElement }
|
||||
|
||||
page.urlsQueue.push({
|
||||
url: urls[i],
|
||||
previewElement
|
||||
})
|
||||
|
||||
function post (i) {
|
||||
if (i === files.length)
|
||||
return done()
|
||||
|
||||
function posted (result) {
|
||||
files[i].previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
|
||||
if (result.success) {
|
||||
page.updateTemplate(files[i], result.files[0])
|
||||
} else {
|
||||
page.updateTemplateIcon(files[i].previewElement, 'icon-block')
|
||||
files[i].previewElement.querySelector('.error').innerHTML = result.description
|
||||
files[i].previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
}
|
||||
|
||||
return post(i + 1)
|
||||
page.processUrlsQueue()
|
||||
document.querySelector('#urls').value = ''
|
||||
}
|
||||
|
||||
files[i].previewElement.querySelector('.descriptive-progress').innerHTML =
|
||||
page.processUrlsQueue = () => {
|
||||
if (!page.urlsQueue.length) return
|
||||
|
||||
function finishedUrlUpload (file, data) {
|
||||
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
|
||||
|
||||
if (data.success === false) {
|
||||
const match = data.description.match(/ over limit: (\d+)$/)
|
||||
if (match && match[1])
|
||||
data.description = `File exceeded limit of ${page.getPrettyBytes(match[1])}.`
|
||||
|
||||
file.previewElement.querySelector('.error').innerHTML = data.description
|
||||
file.previewElement.querySelector('.error').classList.remove('is-hidden')
|
||||
}
|
||||
|
||||
if (Array.isArray(data.files) && data.files[0])
|
||||
page.updateTemplate(file, data.files[0])
|
||||
|
||||
page.activeUrlsQueue--
|
||||
return shiftQueue()
|
||||
}
|
||||
|
||||
function initUrlUpload (file) {
|
||||
file.previewElement.querySelector('.descriptive-progress').innerHTML =
|
||||
'Waiting for server to fetch URL\u2026'
|
||||
|
||||
return axios.post('api/upload', { urls: [files[i].url] }, { headers }).then(response => {
|
||||
return posted(response.data)
|
||||
return axios.post('api/upload', {
|
||||
urls: [file.url]
|
||||
}, {
|
||||
headers: {
|
||||
token: page.token,
|
||||
albumid: page.album,
|
||||
age: page.uploadAge,
|
||||
filelength: page.fileLength
|
||||
}
|
||||
}).catch(error => {
|
||||
return posted({
|
||||
// Format error for display purpose
|
||||
return error.response.data ? error.response : {
|
||||
data: {
|
||||
success: false,
|
||||
description: error.response ? error.response.data.description : error.toString()
|
||||
})
|
||||
description: error.toString()
|
||||
}
|
||||
}
|
||||
}).then(response => {
|
||||
return finishedUrlUpload(file, response.data)
|
||||
})
|
||||
}
|
||||
return post(0)
|
||||
|
||||
function shiftQueue () {
|
||||
while (page.urlsQueue.length && (page.activeUrlsQueue < page.parallelUploads)) {
|
||||
page.activeUrlsQueue++
|
||||
initUrlUpload(page.urlsQueue.shift())
|
||||
}
|
||||
return run()
|
||||
}
|
||||
|
||||
return shiftQueue()
|
||||
}
|
||||
|
||||
page.updateTemplateIcon = (templateElement, iconClass) => {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
{
|
||||
"1": "1574791009",
|
||||
"1": "1575024012",
|
||||
"2": "1568894058",
|
||||
"3": "1568894058",
|
||||
"4": "1568894058",
|
||||
|
||||
52
yarn.lock
52
yarn.lock
@ -203,9 +203,9 @@
|
||||
integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
|
||||
|
||||
"@types/node@*":
|
||||
version "12.12.12"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.12.tgz#529bc3e73dbb35dd9e90b0a1c83606a9d3264bdb"
|
||||
integrity sha512-MGuvYJrPU0HUwqF7LqvIj50RZUX23Z+m583KBygKYUZLlZ88n6w28XRNJRJgsHukLEnLz6w6SvxZoLgbr5wLqQ==
|
||||
version "12.12.14"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.14.tgz#1c1d6e3c75dba466e0326948d56e8bd72a1903d2"
|
||||
integrity sha512-u/SJDyXwuihpwjXy7hOOghagLEV1KdAST6syfnOk6QZAMzZuWZqXy5aYYZbh8Jdpd4escVFP0MvftHNDb9pruA==
|
||||
|
||||
"@types/q@^1.5.1":
|
||||
version "1.5.2"
|
||||
@ -646,9 +646,9 @@ aws-sign2@~0.7.0:
|
||||
integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=
|
||||
|
||||
aws4@^1.8.0:
|
||||
version "1.8.0"
|
||||
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
|
||||
integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
|
||||
version "1.9.0"
|
||||
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.9.0.tgz#24390e6ad61386b0a747265754d2a17219de862c"
|
||||
integrity sha512-Uvq6hVe90D0B2WEnUqtdgY1bATGz3mw33nH9Y+dmA+w5DHvUmBgkr5rM/KCHpCsiFNRUfokW/szpPPgMK2hm4A==
|
||||
|
||||
bach@^1.0.0:
|
||||
version "1.2.0"
|
||||
@ -721,9 +721,9 @@ bl@^3.0.0:
|
||||
readable-stream "^3.0.1"
|
||||
|
||||
bluebird@^3.7.1:
|
||||
version "3.7.1"
|
||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.1.tgz#df70e302b471d7473489acf26a93d63b53f874de"
|
||||
integrity sha512-DdmyoGCleJnkbp3nkbxTLJ18rjDsE4yCggEwKNXkeV123sPNfOCYeDoeuOY+F2FrSjO1YXcTU+dsy96KMy+gcg==
|
||||
version "3.7.2"
|
||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
|
||||
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
|
||||
|
||||
body-parser@1.19.0, body-parser@^1.19.0:
|
||||
version "1.19.0"
|
||||
@ -2216,9 +2216,9 @@ express@^4.17.1:
|
||||
vary "~1.1.2"
|
||||
|
||||
ext@^1.1.2:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/ext/-/ext-1.2.0.tgz#8dd8d2dd21bcced3045be09621fa0cbf73908ba4"
|
||||
integrity sha512-0ccUQK/9e3NreLFg6K6np8aPyRgwycx+oFGtfx1dSp7Wj00Ozw9r05FgBRlzjf2XBM7LAzwgLyDscRrtSU91hA==
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/ext/-/ext-1.3.0.tgz#21526eb296753fed34b620d4a69e3911065fa925"
|
||||
integrity sha512-LErT9cIGZZjSvFkyocVXXeYlj7z8xiA+4oQlM9cX4X/Kfc18cefv3Dd9mNKwFuzUJ7neMMAQz1u1r3gBa/6wGg==
|
||||
dependencies:
|
||||
type "^2.0.0"
|
||||
|
||||
@ -3826,10 +3826,10 @@ kind-of@^6.0.0, kind-of@^6.0.2:
|
||||
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051"
|
||||
integrity sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==
|
||||
|
||||
knex@^0.20.2:
|
||||
version "0.20.2"
|
||||
resolved "https://registry.yarnpkg.com/knex/-/knex-0.20.2.tgz#7429577a95a10f4a4e3090c23b559fed20343b4a"
|
||||
integrity sha512-nw7/RsaZrIGdzbsb1evcEaZv8sL/Ji2W7o5OoF0NIKei4ySU01D4G5mRNVNtneoLoPjUMgqSFRanabhGacJUIA==
|
||||
knex@^0.20.3:
|
||||
version "0.20.3"
|
||||
resolved "https://registry.yarnpkg.com/knex/-/knex-0.20.3.tgz#85178cd6873f75827be86d054c4e117bb4d9657b"
|
||||
integrity sha512-zzYO34pSCCYVqRTbCp8xL+Z7fvHQl5anif3Oacu6JaHFDubB7mFGWRRJBNSO3N8Ql4g4CxUgBctaPiliwoOsNA==
|
||||
dependencies:
|
||||
bluebird "^3.7.1"
|
||||
colorette "1.1.0"
|
||||
@ -4414,9 +4414,9 @@ nocache@2.1.0:
|
||||
integrity sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==
|
||||
|
||||
node-abi@^2.7.0:
|
||||
version "2.12.0"
|
||||
resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.12.0.tgz#40e9cfabdda1837863fa825e7dfa0b15686adf6f"
|
||||
integrity sha512-VhPBXCIcvmo/5K8HPmnWJyyhvgKxnHTUMXR/XwGHV68+wrgkzST4UmQrY/XszSWA5dtnXpNp528zkcyJ/pzVcw==
|
||||
version "2.13.0"
|
||||
resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.13.0.tgz#e2f2ec444d0aca3ea1b3874b6de41d1665828f63"
|
||||
integrity sha512-9HrZGFVTR5SOu3PZAnAY2hLO36aW1wmA+FDsVkr85BTST32TLCA1H/AEcatVRAsWLyXS3bqUDYCAjq5/QGuSTA==
|
||||
dependencies:
|
||||
semver "^5.4.1"
|
||||
|
||||
@ -6179,9 +6179,9 @@ resolve-url@^0.2.1:
|
||||
integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=
|
||||
|
||||
resolve@^1.1.6, resolve@^1.1.7, resolve@^1.10.0, resolve@^1.10.1, resolve@^1.11.0, resolve@^1.3.2, resolve@^1.4.0, resolve@^1.5.0:
|
||||
version "1.12.2"
|
||||
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.12.2.tgz#08b12496d9aa8659c75f534a8f05f0d892fff594"
|
||||
integrity sha512-cAVTI2VLHWYsGOirfeYVVQ7ZDejtQ9fp4YhYckWDEkFfqbVjaT11iM8k6xSAfGFMM+gDpZjMnFssPu8we+mqFw==
|
||||
version "1.13.1"
|
||||
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.13.1.tgz#be0aa4c06acd53083505abb35f4d66932ab35d16"
|
||||
integrity sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==
|
||||
dependencies:
|
||||
path-parse "^1.0.6"
|
||||
|
||||
@ -6892,10 +6892,10 @@ svgo@^1.0.0:
|
||||
unquote "~1.1.1"
|
||||
util.promisify "~1.0.0"
|
||||
|
||||
systeminformation@^4.15.3:
|
||||
version "4.15.3"
|
||||
resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-4.15.3.tgz#639cdc224b5c3811f1a0bfba33f869bbc6fb930f"
|
||||
integrity sha512-Fx2ARGHtLl2/xLeNoTR8/doXSxUXuAzIN+dyCK9O43j/UETLBt77yTEbTxmYsVD47PYjX1iQTdcY41CZckY+zg==
|
||||
systeminformation@^4.16.0:
|
||||
version "4.16.0"
|
||||
resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-4.16.0.tgz#d92ce821efdab720496c60656bf65cc68fb03f8c"
|
||||
integrity sha512-1FjxPJSw7ad0zug+1YIQATj6Cn+wM5OBASEpjohEeOD2EGPIf0Cnhthd1L2O1YX+wKgOMuPldGfxYdo8yNHEIg==
|
||||
|
||||
table@^5.2.3:
|
||||
version "5.4.6"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user