Initialize upload filters for regular users (WIP)

Updated ESLint's ECMA version to 9 (2018).
I'll need to use some lookbehind regex directives from now on.
It's supported since Node 10, which is the oldest version I'll support.

Refactored "can not" -> "cannot".

Filtering for regular users is still work in progress.
Some features aren't working as expected yet.
This commit is contained in:
Bobby Wibowo 2020-05-03 02:39:24 +07:00
parent 0eb425e216
commit 1980d536db
No known key found for this signature in database
GPG Key ID: 51C3A1E1E22D26CF
8 changed files with 288 additions and 154 deletions

View File

@ -1,7 +1,7 @@
{
"root": true,
"parserOptions": {
"ecmaVersion": 8
"ecmaVersion": 9
},
"env": {
"node": true

View File

@ -761,11 +761,16 @@ self.list = async (req, res) => {
const filters = req.headers.filters
const minoffset = req.headers.minoffset
const ismoderator = perms.is(user, 'moderator')
if ((all || filters) && !ismoderator)
if (all && !ismoderator)
return res.status(403).end()
const basedomain = config.domain
// Thresholds for regular users
const MAX_WILDCARDS_IN_KEY = 2
const MAX_TEXT_QUERIES = 3 // non-keyed keywords
const MAX_SORT_KEYS = 1
const filterObj = {
uploaders: [],
excludeUploaders: [],
@ -794,15 +799,47 @@ self.list = async (req, res) => {
parsed: []
}
// Parse glob wildcards into SQL wildcards
function sqlLikeParser (pattern) {
// Escape SQL operators
const escaped = pattern
.replace(/(?<!\\)%/g, '\\%')
.replace(/(?<!\\)_/g, '\\_')
// Look for any glob operators
const match = pattern.match(/(?<!\\)(\*|\?)/g)
if (match && match.length) {
logger.debug(pattern, match)
return {
count: match.length,
// Replace glob operators with their SQL equivalents
escaped: escaped
.replace(/(?<!\\)\*/g, '%')
.replace(/(?<!\\)\?/g, '_')
}
} else {
return {
count: 0,
// Assume partial match
escaped: `%${escaped}%`
}
}
}
if (filters) {
const keywords = [
let keywords = []
// Only allow filtering by 'ip' and 'user' keys when listing all uploads
if (all)
keywords = keywords.concat([
'ip',
'user'
]
])
const ranges = [
'date',
'expiry'
]
filterObj.queries = searchQuery.parse(filters, {
keywords: keywords.concat([
'sort'
@ -813,6 +850,45 @@ self.list = async (req, res) => {
offsets: false
})
// For some reason, single value won't be in Array even with 'alwaysArray' option
if (typeof filterObj.queries.exclude.text === 'string')
filterObj.queries.exclude.text = [filterObj.queries.exclude.text]
let textQueries = 0
if (filterObj.queries.text) textQueries += filterObj.queries.text.length
if (filterObj.queries.exclude.text) textQueries += filterObj.queries.exclude.text.length
// Regular user threshold check
if (!ismoderator && textQueries > MAX_TEXT_QUERIES)
return res.json({
success: false,
description: `Users are only allowed to use ${MAX_TEXT_QUERIES} non-keyed keyword${MAX_TEXT_QUERIES === 1 ? '' : 's'} at a time.`
})
if (filterObj.queries.text)
for (let i = 0; i < filterObj.queries.text.length; i++) {
const result = sqlLikeParser(filterObj.queries.text[i])
if (result.count > MAX_WILDCARDS_IN_KEY)
return res.json({
success: false,
description: `Users are only allowed to use ${MAX_WILDCARDS_IN_KEY} wildcard${MAX_WILDCARDS_IN_KEY === 1 ? '' : 's'} per key.`
})
filterObj.queries.text[i] = result.escaped
}
if (filterObj.queries.exclude.text) {
textQueries += filterObj.queries.exclude.text.length
for (let i = 0; i < filterObj.queries.exclude.text.length; i++) {
const result = sqlLikeParser(filterObj.queries.exclude.text[i])
if (result.count > MAX_WILDCARDS_IN_KEY)
return res.json({
success: false,
description: `Users are only allowed to use ${MAX_WILDCARDS_IN_KEY} wildcard${MAX_WILDCARDS_IN_KEY === 1 ? '' : 's'} per key.`
})
filterObj.queries.exclude.text[i] = result.escaped
}
}
for (const key of keywords) {
let queryIndex = -1
let excludeIndex = -1
@ -833,8 +909,18 @@ self.list = async (req, res) => {
if (inQuery || inExclude) {
// Prioritize exclude keys when both types found
filterObj.flags[`${key}Null`] = inExclude ? false : inQuery
if (inQuery) filterObj.queries[key].splice(queryIndex, 1)
if (inExclude) filterObj.queries.exclude[key].splice(excludeIndex, 1)
if (inQuery)
if (filterObj.queries[key].length === 1)
// Delete key to avoid unexpected behavior
delete filterObj.queries[key]
else
filterObj.queries[key].splice(queryIndex, 1)
if (inExclude)
if (filterObj.queries.exclude[key].length === 1)
// Delete key to avoid unexpected behavior
delete filterObj.queries.exclude[key]
else
filterObj.queries.exclude[key].splice(excludeIndex, 1)
}
}
@ -909,45 +995,69 @@ self.list = async (req, res) => {
else
filterObj.excludeUploaders.push(uploader)
// Delete keys to avoid unexpected behavior
delete filterObj.queries.user
delete filterObj.queries.exclude.user
}
// Parse sort keys
if (filterObj.queries.sort) {
let allowed = [
'albumid',
'expirydate',
'id',
'name',
'size',
'timestamp'
]
// Only allow sorting by 'ip' and 'userid' columns when listing all uploads
if (all)
allowed = allowed.concat([
'ip',
'userid'
])
for (const obQuery of filterObj.queries.sort) {
const tmp = obQuery.toLowerCase().split(':')
const column = sortObj.maps[tmp[0]] || tmp[0]
let column = sortObj.maps[tmp[0]] || tmp[0]
let direction = 'asc'
if (!allowed.includes(column))
// Alert users if using disallowed/missing columns
return res.json({ success: false, description: `Column \`${column}\` cannot be used for sorting.\n\nTry the following instead:\n${allowed.join(', ')}` })
if (sortObj.casts[column])
column = `cast (\`${column}\` as ${sortObj.casts[column]})`
if (tmp[1] && /^d/.test(tmp[1]))
direction = 'desc'
const suffix = sortObj.nullsLast.includes(column) ? ' nulls last' : ''
sortObj.parsed.push(`${column} ${direction}${suffix}`)
sortObj.parsed.push({
column,
order: (tmp[1] && /^d/.test(tmp[1])) ? 'desc' : 'asc',
clause: sortObj.nullsLast.includes(column) ? 'nulls last' : '',
cast: sortObj.casts[column] || null
})
}
// Regular user threshold check
if (!ismoderator && sortObj.parsed.length > MAX_SORT_KEYS)
return res.json({
success: false,
description: `Users are only allowed to use ${MAX_SORT_KEYS} sort key${MAX_SORT_KEYS === 1 ? '' : 's'} at a time.`
})
// Delete key to avoid unexpected behavior
delete filterObj.queries.sort
}
// For some reason, single value won't be in Array even with 'alwaysArray' option
if (typeof filterObj.queries.exclude.text === 'string')
filterObj.queries.exclude.text = [filterObj.queries.exclude.text]
}
function filter () {
if (req.params.id !== undefined)
if (req.params.id !== undefined) {
this.where('albumid', req.params.id)
else if (!all)
} else {
// Sheesh, these look so overbearing...
if (!all)
// If not listing all uploads, list user's uploads
this.where('userid', user.id)
else
// Sheesh, these look too overbearing...
this.where(function () {
// Filter uploads matching any of the supplied 'user' keys and/or NULL flag
// Prioritze exclude keys when both types found
this.orWhere(function () {
if (filterObj.excludeUploaders.length)
this.orWhereNotIn('userid', filterObj.excludeUploaders.map(v => v.id))
else if (filterObj.uploaders.length)
@ -959,9 +1069,11 @@ self.list = async (req, res) => {
this.orWhereNull('userid')
else if (filterObj.flags.userNull === false)
this.orWhereNotNull('userid')
}).orWhere(function () {
})
// Filter uploads matching any of the supplied 'ip' keys and/or NULL flag
// Same prioritization logics as above
// Same prioritization logic as above
this.orWhere(function () {
if (filterObj.queries.exclude.ip)
this.orWhereNotIn('ip', filterObj.queries.exclude.ip)
else if (filterObj.queries.ip)
@ -973,9 +1085,12 @@ self.list = async (req, res) => {
this.orWhereNull('ip')
else if (filterObj.flags.ipNull === false)
this.orWhereNotNull('ip')
}).andWhere(function () {
// Then, refine using the supplied 'date' and/or 'expiry' ranges
if (filterObj.queries.date)
})
})
// Then, refine using the supplied 'date' ranges
this.andWhere(function () {
if (!filterObj.queries.date) return
if (typeof filterObj.queries.date.from === 'number')
if (typeof filterObj.queries.date.to === 'number')
this.andWhereBetween('timestamp', [filterObj.queries.date.from, filterObj.queries.date.to])
@ -983,7 +1098,11 @@ self.list = async (req, res) => {
this.andWhere('timestamp', '>=', filterObj.queries.date.from)
else
this.andWhere('timestamp', '<=', filterObj.queries.date.to)
if (filterObj.queries.expiry)
})
// Then, refine using the supplied 'expiry' ranges
this.andWhere(function () {
if (!filterObj.queries.expiry) return
if (typeof filterObj.queries.expiry.from === 'number')
if (typeof filterObj.queries.expiry.to === 'number')
this.andWhereBetween('expirydate', [filterObj.queries.expiry.from, filterObj.queries.expiry.to])
@ -991,25 +1110,22 @@ self.list = async (req, res) => {
this.andWhere('expirydate', '>=', filterObj.queries.date.from)
else
this.andWhere('expirydate', '<=', filterObj.queries.date.to)
}).andWhere(function () {
// Then, refine using the supplied keywords against their file names
if (!filterObj.queries.text) return
for (const name of filterObj.queries.text)
if (name.includes('*'))
this.orWhere('name', 'like', name.replace(/\*/g, '%'))
else
// If no asterisks, assume partial
this.orWhere('name', 'like', `%${name}%`)
}).andWhere(function () {
// Finally, refine using the supplied exclusions against their file names
if (!filterObj.queries.exclude.text) return
for (const exclude of filterObj.queries.exclude.text)
if (exclude.includes('*'))
this.andWhere('name', 'not like', exclude.replace(/\*/g, '%'))
else
// If no asterisks, assume partial
this.andWhere('name', 'not like', `%${exclude}%`)
})
// Then, refine using the supplied keywords against their file names
this.andWhere(function () {
if (!filterObj.queries.text) return
for (const pattern of filterObj.queries.text)
this.orWhere('name', 'like', pattern)
})
// Finally, refine using the supplied exclusions against their file names
this.andWhere(function () {
if (!filterObj.queries.exclude.text) return
for (const pattern of filterObj.queries.exclude.text)
this.orWhere('name', 'not like', pattern)
})
}
}
try {
@ -1032,12 +1148,22 @@ self.list = async (req, res) => {
// Only select IPs if we are listing all uploads
columns.push(all ? 'ip' : 'albumid')
const sortRaw = sortObj.parsed.length
? sortObj.parsed.join(', ')
: '`id` desc'
// Build raw query for order by (sorting) operation
let orderByRaw
if (sortObj.parsed.length)
orderByRaw = sortObj.parsed.map(sort => {
// Use Knex.raw() to sanitize user inputs
if (sort.cast)
return db.raw(`cast (?? as ${sort.cast}) ${sort.order} ${sort.clause}`.trim(), sort.column)
else
return db.raw(`?? ${sort.order} ${sort.clause}`.trim(), sort.column)
}).join(', ')
else
orderByRaw = '`id` desc'
const files = await db.table('files')
.where(filter)
.orderByRaw(sortRaw)
.orderByRaw(orderByRaw)
.limit(25)
.offset(25 * offset)
.select(columns)

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -457,7 +457,7 @@ page.getUploads = (params = {}) => {
if (params === undefined)
params = {}
if ((params.all || params.filters) && !page.permissions.moderator)
if (params.all && !page.permissions.moderator)
return swal('An error occurred!', 'You cannot do this!', 'error')
page.updateTrigger(params.trigger, 'loading')
@ -474,9 +474,9 @@ page.getUploads = (params = {}) => {
filters: params.filters || ''
}
// Send client timezone offset if using date filter
// Send client timezone offset if using filters
// Server will pretend client is on UTC if missing
if (headers.filters.includes('date:') || headers.filters.includes('expiry:'))
if (headers.filters)
headers.minOffset = new Date().getTimezoneOffset()
axios.get(url, { headers }).then(response => {
@ -507,9 +507,7 @@ page.getUploads = (params = {}) => {
const basedomain = response.data.basedomain
const pagination = page.paginate(response.data.count, 25, params.pageNum)
let filter = '<div class="column is-hidden-mobile"></div>'
if (params.all)
filter = `
const filter = `
<div class="column">
<form class="prevent-default">
<div class="field has-addons">
@ -517,7 +515,7 @@ page.getUploads = (params = {}) => {
<input id="filters" class="input is-small" type="text" placeholder="Filters" value="${page.escape(params.filters || '')}">
</div>
<div class="control">
<button type="button" class="button is-small is-primary is-outlined" title="Help?" data-action="upload-filters-help">
<button type="button" class="button is-small is-primary is-outlined" title="Help?" data-action="upload-filters-help"${params.all ? ' data-all="true"' : ''}">
<span class="icon">
<i class="icon-help-circled"></i>
</span>
@ -1012,9 +1010,10 @@ page.clearSelection = () => {
}
page.uploadFiltersHelp = element => {
const all = Boolean(element.dataset.all)
const content = document.createElement('div')
content.style = 'text-align: left'
content.innerHTML = `
content.innerHTML = `${all ? `
There are 2 filter keys, namely <b>user</b> (username) and <b>ip</b>.
These keys can be specified more than once.
For usernames with whitespaces, wrap them with double quotes (<code>"</code>).
@ -1022,10 +1021,12 @@ page.uploadFiltersHelp = element => {
To exclude certain users/ips while still listing every other uploads, add negation sign (<code>-</code>) before the keys.
Negation sign can also be used to exclude the special cases mentioned above (i.e. <code>-user:-</code> or <code>-ip:-</code>).
` : ''}
There are 2 range keys: <b>date</b> (upload date) and <b>expiry</b> (expiry date).
Their format is: <code>YYYY/MM/DD HH:MM:SS-YYYY/MM/DD HH:MM:SS</code> ("from" date and "to" date respectively).
You can specify only one date. If "to" date is missing, 'now' will be used. If "from" date is missing, 'beginning of time' will be used.
You may specify only one of the dates.
If "to" date is missing, 'now' will be used.
If "from" date is missing, 'beginning of time' will be used.
If any of the subsequent date or time units are not specified, their first value will be used (e.g. January for month, 1 for day, and so on).
If only time is specified, today's date will be used.
In conclusion, the following examples are all valid: <code>date:2020/01/01 01:23-2018/01/01 06</code>, <code>expiry:-2020/05</code>, <code>date:12:34:56</code>.
@ -1036,41 +1037,44 @@ page.uploadFiltersHelp = element => {
Matches can also be sorted with <b>sort</b> keys.
Its format is: <code>sort:columnName[:d[escending]]</code>, where <code>:d[escending]</code> is an optional tag to set the direction to descending.
This key must be used with internal column names used in the database (<code>id</code>, <code>userid</code>, and so on),
This key must be used with internal column names used in the database (<code>id</code>, <code>${all ? 'userid' : 'albumid'}</code>, and so on),
but there are 2 shortcuts available: <b>date</b> for <code>timestamp</code> column and <b>expiry</b> for <code>expirydate</code> column.
This key can also be specified more than once, where their order will decide the sorting steps.
Any leftover keywords which do not use keys (non-keyed keywords) will be matched against the matches' file names.
Excluding certain keywords is also supported by adding negation sign (<b>-</b>) before the keywords.
<b>Internals:</b>
First, query uploads passing ALL exclusion filter keys OR matching ANY filter keys, if any.
Second, refine matches using range keys, if any.
Third, refine matches using ANY non-keyed keywords, if any.
Fourth, filter matches using ALL exclusion non-keyed keywords, if any.
Fifth, sort matches using sorting keys, if any.
<b>Internal steps:</b>
${all ? `- Query uploads passing ALL exclusion filter keys OR matching ANY filter keys, if any.
- Refine matches` : '- Filter uploads'} using date key, if any.
- Refine matches using expiry key, if any.
- Refine matches using ANY non-keyed keywords, if any.
- Filter matches using ALL exclusion non-keyed keywords, if any.
- Sort matches using sorting keys, if any.
<b>Examples:</b>
Uploads from users named "demo" AND/OR "John Doe" AND/OR non-registered users:
${all ? `- Uploads from users named "demo" AND/OR "John Doe" AND/OR non-registered users:
<code>user:demo user:"John Doe" user:-</code>
ALL uploads, but NOT the ones from user named "demo" AND "John Doe":
- ALL uploads, but NOT the ones from user named "demo" AND "John Doe":
<code>-user:demo -user:"John Doe"</code>
Uploads from IP "127.0.0.1" AND which file names match "*.rar" OR "*.zip":
- Uploads from IP "127.0.0.1" AND which file names match "*.rar" OR "*.zip":
<code>ip:127.0.0.1 *.rar *.zip</code>
Uploads uploaded since "1 June 2019 00:00:00":
` : ''}- Uploads uploaded since "1 June 2019 00:00:00":
<code>date:2019/06</code>
Uploads uploaded between "7 April 2020 12:00:00" and "7 April 2020 23:59:59":
- Uploads uploaded between "7 April 2020 12:00:00" and "7 April 2020 23:59:59":
<code>date:2020/04/07 12-2020/04/07 23:59:59</code>
Uploads uploaded before "5 February 2020 00:00:00":
- Uploads uploaded before "5 February 2020 00:00:00":
<code>date:-2020/02/05</code>
Uploads which file names match "*.gz" but NOT "*.tar.gz":
- Uploads which file names match "*.gz" but NOT "*.tar.gz":
<code>*.gz -*.tar.gz</code>
Sort matches by "size" column in ascending and descending order respectively:
<code>user:"John Doe" sort:size</code>
<code>*.mp4 user:- sort:size:d</code>
- Sort matches by "size" column in ascending and descending order respectively:
<code>${all ? 'user:"John Doe"' : '*.txt'} sort:size</code>
<code>*.mp4 ${all ? 'user:- ' : ''}sort:size:d</code>
${!page.permissions.moderator ? `
<b>Notice:</b> Regular users may face some limitations in the amount of keys that can be used at a time.
` : ''}
<b>Friendly reminder:</b> This window can be scrolled up!
`.trim().replace(/^ {6}/gm, '').replace(/\n/g, '<br>')
`.trim().replace(/^\s*/g, '').replace(/\n/g, '<br>')
swal({ content }).then(() => {
// Restore modal size
@ -1083,7 +1087,11 @@ page.uploadFiltersHelp = element => {
page.filterUploads = element => {
const filters = document.querySelector(`#${element.dataset.filtersid || 'filters'}`).value.trim()
page.getUploads({ all: true, filters }, element)
// eslint-disable-next-line compat/compat
page.getUploads(Object.assign(page.views[page.currentView], {
filters,
pageNum: 0
}), element)
}
page.viewUserUploads = (id, element) => {
@ -2170,7 +2178,7 @@ page.createUser = () => {
content: div
})
// Reload users list
// Load last page of users list
// eslint-disable-next-line compat/compat
page.getUsers(Object.assign(page.views.users, {
pageNum: -1

View File

@ -1,5 +1,5 @@
{
"1": "1588434390",
"1": "1588448172",
"2": "1581416390",
"3": "1581416390",
"4": "1581416390",