Mirror/filesafe
1
0
Fork 0
mirror of https://github.com/BobbyWibowo/lolisafe.git synced 2025-02-22 05:09:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

More improvements to uploads filtering!

Browse Source 
Renamed "orderby" key to "sort" (for sorting uploads).

Fixed non-keyed keyword exclusions not working as expected when
more than one are used at the same time.

Support not specifying "from" date when filtering with range keys
(date and expiry).

Proper logic for NULL values inclusion/exclusion when filtering with
user and/or ip keys.

Improved Help? prompt again!!
Also clarify about timezone differences.

Added logger.debug() function.
Basically a shorthand for console.log(require('util').inspect()).

Rebuilt client asssets and bumped v1 version string.
This commit is contained in:
Bobby Wibowo 2020-04-20 01:19:20 +07:00
parent dc59476592
commit 922269181c
No known key found for this signature in database
GPG Key ID: 51C3A1E1E22D26CF
6 changed files with 104 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
controllers/uploadController.js
View File

@ -775,7 +775,7 @@ self.list = async (req, res) => {
flags: {}
}
const orderByObj = {
const sortObj = {
// Cast columns to specific type if they are stored differently
casts: {
size: 'integer'
@ -805,7 +805,7 @@ self.list = async (req, res) => {
]
filterObj.queries = searchQuery.parse(filters, {
keywords: keywords.concat([
'orderby'
'sort'
]),
ranges,
tokenize: true,
@ -813,18 +813,30 @@ self.list = async (req, res) => {
offsets: false
})
for (const key of keywords)
if (filterObj.queries[key]) {
// Make sure keyword arrays only contain unique values
filterObj.queries[key] = filterObj.queries[key].filter((v, i, a) => a.indexOf(v) === i)
for (const key of keywords) {
let queryIndex = -1
let excludeIndex = -1
// Flag to match NULL values
const index = filterObj.queries[key].indexOf('-')
if (index !== -1) {
filterObj.flags[`no${key}`] = true
filterObj.queries[key].splice(index, 1)
}
// Make sure keyword arrays only contain unique values
if (filterObj.queries[key]) {
filterObj.queries[key] = filterObj.queries[key].filter((v, i, a) => a.indexOf(v) === i)
queryIndex = filterObj.queries[key].indexOf('-')
}
if (filterObj.queries.exclude[key]) {
filterObj.queries.exclude[key] = filterObj.queries.exclude[key].filter((v, i, a) => a.indexOf(v) === i)
excludeIndex = filterObj.queries.exclude[key].indexOf('-')
}
// Flag to match NULL values
const inQuery = queryIndex !== -1
const inExclude = excludeIndex !== -1
if (inQuery || inExclude) {
// Prioritize exclude keys when both types found
filterObj.flags[`${key}Null`] = inExclude ? false : inQuery
if (inQuery) filterObj.queries[key].splice(queryIndex, 1)
if (inExclude) filterObj.queries.exclude[key].splice(excludeIndex, 1)
}
}
const parseDate = (date, minoffset, resetMs) => {
// [YYYY][/MM][/DD] [HH][:MM][:SS]
@ -901,24 +913,24 @@ self.list = async (req, res) => {
delete filterObj.queries.exclude.user
}
// Parse orderby keys
if (filterObj.queries.orderby) {
for (const obQuery of filterObj.queries.orderby) {
// Parse sort keys
if (filterObj.queries.sort) {
for (const obQuery of filterObj.queries.sort) {
const tmp = obQuery.toLowerCase().split(':')
let column = orderByObj.maps[tmp[0]] || tmp[0]
let column = sortObj.maps[tmp[0]] || tmp[0]
let direction = 'asc'
if (orderByObj.casts[column])
column = `cast (\`${column}\` as ${orderByObj.casts[column]})`
if (sortObj.casts[column])
column = `cast (\`${column}\` as ${sortObj.casts[column]})`
if (tmp[1] && /^d/.test(tmp[1]))
direction = 'desc'
const suffix = orderByObj.nullsLast.includes(column) ? ' nulls last' : ''
orderByObj.parsed.push(`${column} ${direction}${suffix}`)
const suffix = sortObj.nullsLast.includes(column) ? ' nulls last' : ''
sortObj.parsed.push(`${column} ${direction}${suffix}`)
}
delete filterObj.queries.orderby
delete filterObj.queries.sort
}
// For some reason, single value won't be in Array even with 'alwaysArray' option
@ -935,32 +947,50 @@ self.list = async (req, res) => {
// Sheesh, these look too overbearing...
this.where(function () {
// Filter uploads matching any of the supplied 'user' keys and/or NULL flag
if (filterObj.uploaders.length)
this.orWhereIn('userid', filterObj.uploaders.map(v => v.id))
// Prioritze exclude keys when both types found
if (filterObj.excludeUploaders.length)
this.orWhereNotIn('userid', filterObj.excludeUploaders.map(v => v.id))
if (filterObj.flags.nouser)
else if (filterObj.uploaders.length)
this.orWhereIn('userid', filterObj.uploaders.map(v => v.id))
// Such overbearing logic for NULL values, smh...
if ((filterObj.excludeUploaders.length && filterObj.flags.userNull !== false) ||
(filterObj.uploaders.length && filterObj.flags.userNull) ||
(!filterObj.excludeUploaders.length && !filterObj.uploaders.length && filterObj.flags.userNull))
this.orWhereNull('userid')
else if (filterObj.flags.userNull === false)
this.orWhereNotNull('userid')
}).orWhere(function () {
// Filter uploads matching any of the supplied 'ip' keys and/or NULL flag
if (filterObj.queries.ip)
this.orWhereIn('ip', filterObj.queries.ip)
// Same prioritization logics as above
if (filterObj.queries.exclude.ip)
this.orWhereNotIn('ip', filterObj.queries.exclude.ip)
if (filterObj.flags.noip)
else if (filterObj.queries.ip)
this.orWhereIn('ip', filterObj.queries.ip)
// ...
if ((filterObj.queries.exclude.ip && filterObj.flags.ipNull !== false) ||
(filterObj.queries.ip && filterObj.flags.ipNull) ||
(!filterObj.queries.exclude.ip && !filterObj.queries.ip && filterObj.flags.ipNull))
this.orWhereNull('ip')
else if (filterObj.flags.ipNull === false)
this.orWhereNotNull('ip')
}).andWhere(function () {
// Then, refine using the supplied 'date' and/or 'expiry' ranges
if (filterObj.queries.date)
if (typeof filterObj.queries.date.to === 'number')
this.andWhereBetween('timestamp', [filterObj.queries.date.from, filterObj.queries.date.to])
if (typeof filterObj.queries.date.from === 'number')
if (typeof filterObj.queries.date.to === 'number')
this.andWhereBetween('timestamp', [filterObj.queries.date.from, filterObj.queries.date.to])
else
this.andWhere('timestamp', '>=', filterObj.queries.date.from)
else
this.andWhere('timestamp', '>=', filterObj.queries.date.from)
this.andWhere('timestamp', '<=', filterObj.queries.date.to)
if (filterObj.queries.expiry)
if (typeof filterObj.queries.expiry.to === 'number')
this.andWhereBetween('expirydate', [filterObj.queries.expiry.from, filterObj.queries.expiry.to])
if (typeof filterObj.queries.expiry.from === 'number')
if (typeof filterObj.queries.expiry.to === 'number')
this.andWhereBetween('expirydate', [filterObj.queries.expiry.from, filterObj.queries.expiry.to])
else
this.andWhere('expirydate', '>=', filterObj.queries.date.from)
else
this.andWhere('expirydate', '>=', filterObj.queries.date.from)
this.andWhere('expirydate', '<=', filterObj.queries.date.to)
}).andWhere(function () {
// Then, refine using the supplied keywords against their file names
if (!filterObj.queries.text) return
@ -975,10 +1005,10 @@ self.list = async (req, res) => {
if (!filterObj.queries.exclude.text) return
for (const exclude of filterObj.queries.exclude.text)
if (exclude.includes('*'))
this.orWhere('name', 'not like', exclude.replace(/\*/g, '%'))
this.andWhere('name', 'not like', exclude.replace(/\*/g, '%'))
else
// If no asterisks, assume partial
this.orWhere('name', 'not like', `%${exclude}%`)
this.andWhere('name', 'not like', `%${exclude}%`)
})
}
@ -1001,12 +1031,12 @@ self.list = async (req, res) => {
// Only select IPs if we are listing all uploads
columns.push(all ? 'ip' : 'albumid')
const orderByRaw = orderByObj.parsed.length
? orderByObj.parsed.join(', ')
const sortRaw = sortObj.parsed.length
? sortObj.parsed.join(', ')
: '`id` desc'
const files = await db.table('files')
.where(filter)
.orderByRaw(orderByRaw)
.orderByRaw(sortRaw)
.limit(25)
.offset(25 * offset)
.select(columns)

2
dist/js/dashboard.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/js/dashboard.js.map vendored
View File

File diff suppressed because one or more lines are too long

5
logger.js
View File

@ -45,4 +45,9 @@ self.error = (content, options = {}) => {
write(content, options)
}
self.debug = (...args) => {
for (const arg of args)
console.log(inspect(arg, { depth: Infinity }))
}
module.exports = self

50
src/js/dashboard.js
View File

@ -969,51 +969,57 @@ page.uploadFiltersHelp = element => {
content.innerHTML = `
There are 2 filter keys, namely <b>user</b> (username) and <b>ip</b>.
These keys can be specified more than once.
For usernames with whitespaces, wrap them with double quotes (<b>"</b>).
For usernames with whitespaces, wrap them with double quotes (<code>"</code>).
Special cases such as uploads by non-registered users or have no IPs respectively, use <code>user:-</code> or <code>ip:-</code>.
There are 2 special filter keys, namely <b>user:-</b> and <b>ip:-</b>, to match uploads by non-registered users and have no IPs respectively.
To exclude certain users/ips while still listing every other uploads, add negation sign (<b>-</b>) before the keys.
<b>Unfortunately</b>, this will not show uploads by non-registered users or have no IPs as well, instead you need to also use the special filter keys if you want to include them.
To exclude certain users/ips while still listing every other uploads, add negation sign (<code>-</code>) before the keys.
Negation sign can also be used to exclude the special cases mentioned above (i.e. <code>-user:-</code> or <code>-ip:-</code>).
There are 2 range keys: <b>date</b> (upload date) and <b>expiry</b> (expiry date).
Their format is: <b>YYYY/MM/DD HH:MM:SS-YYYY/MM/DD HH:MM:SS</b> ("to" date is optional).
Their format is: <code>YYYY/MM/DD HH:MM:SS-YYYY/MM/DD HH:MM:SS</code> ("from" date and "to" date respectively).
You can specify only one date. If "to" date is missing, 'now' will be used. If "from" date is missing, 'beginning of time' will be used.
If any of the subsequent date or time units are not specified, their first value will be used (e.g. January for month, 1 for day, and so on).
If only time is specified, today's date will be used.
Meaning the following can be accepted: <b>2020/01/01 01:23</b>, <b>2018/01/01 06</b>, <b>2019/11</b>, <b>12:34:56</b>.
In conclusion, the following examples are all valid: <code>date:2020/01/01 01:23-2018/01/01 06</code>, <code>expiry:-2020/05</code>, <code>date:12:34:56</code>.
These keys can only be specified once each.
Matches can also be sorted with <b>orderby:columnName[:d[escending]]</b> keys.
This key require internal column names used in the database (id, userid, and so on), but there are 2 shortcuts, namely <b>date</b> for timestamp column and <b>expiry</b> for expirydate column.
<b>Timezone?</b> Don't fret, feel free to query the dates with your own timezone!
API requests to the filter endpoint will attach your browser's timezone offset, so the server will automatically calculate timezone differences.
Matches can also be sorted with <b>sort</b> keys.
Its format is: <code>sort:columnName[:d[escending]]</code>, where <code>:d[escending]</code> is an optional tag to set the direction to descending.
This key must be used with internal column names used in the database (<code>id</code>, <code>userid</code>, and so on),
but there are 2 shortcuts available: <b>date</b> for <code>timestamp</code> column and <b>expiry</b> for <code>expirydate</code> column.
This key can also be specified more than once, where their order will decide the sorting steps.
Any leftover keywords which do not use keys will be matched against the matches' file names.
Any leftover keywords which do not use keys (non-keyed keywords) will be matched against the matches' file names.
Excluding certain keywords is also supported by adding negation sign (<b>-</b>) before the keywords.
<b>Internals:</b>
First, it will filter uploads matching ANY of the supplied filter keys AND/OR special filter keys, if any.
Second, it will refine the matches using the supplied <b>date</b> AND/OR <b>expiry</b> range keys, if any.
Third, it will refine the matches using the leftover non-keyed keywords, if any.
Finally, it will sort the matches using the supplied <b>orderby</b> keys, if any.
First, query uploads passing ALL exclusion filter keys OR matching ANY filter keys, if any.
Second, refine matches using range keys, if any.
Third, refine matches using ANY non-keyed keywords, if any.
Fourth, filter matches using ALL exclusion non-keyed keywords, if any.
Fifth, sort matches using sorting keys, if any.
<b>Examples:</b>
Uploads from user named "demo":
<code>user:demo</code>
Uploads from users named "demo" AND/OR "John Doe" AND/OR non-registered users:
<code>user:demo user:"John Doe" user:-</code>
ALL uploads, including from non-registered users, but NOT the ones from user named "demo":
<code>-user:demo user:-</code>
ALL uploads, but NOT the ones from user named "demo" AND "John Doe":
<code>-user:demo -user:"John Doe"</code>
Uploads from IP "127.0.0.1" AND which file names match "*.rar" OR "*.zip":
<code>ip:127.0.0.1 *.rar *.zip</code>
Uploads uploaded since "1 June 2019 00:00:00":
<code>date:2019/06</code>
Uploads uploaded between "7 April 2020 00:00:00" and "7 April 2020 23:59:59":
<code>date:2020/04/07-2020/04/07 23:59:59</code>
Uploads uploaded between "7 April 2020 12:00:00" and "7 April 2020 23:59:59":
<code>date:2020/04/07 12-2020/04/07 23:59:59</code>
Uploads uploaded before "5 February 2020 00:00:00":
<code>date:-2020/02/05</code>
Uploads which file names match "*.gz" but NOT "*.tar.gz":
<code>*.gz -*.tar.gz</code>
Sort matches by "size" column in ascending and descending order respectively:
<code>user:"John Doe" orderby:size</code>
<code>*.mp4 user:- orderby:size:d</code>
<code>user:"John Doe" sort:size</code>
<code>*.mp4 user:- sort:size:d</code>
<b>Friendly reminder:</b> This window can be scrolled up!
`.trim().replace(/^ {6}/gm, '').replace(/\n/g, '<br>')

2
src/versions.json
View File

@ -1,5 +1,5 @@
{
"1": "1587307215",
"1": "1587320065",
"2": "1581416390",
"3": "1581416390",
"4": "1581416390",