More improvements to uploads filtering!

Renamed "orderby" key to "sort" (for sorting uploads).

Fixed non-keyed keyword exclusions not working as expected when
more than one are used at the same time.

Support not specifying "from" date when filtering with range keys
(date and expiry).

Proper logic for NULL values inclusion/exclusion when filtering with
user and/or ip keys.

Improved Help? prompt again!!
Also clarify about timezone differences.

Added logger.debug() function.
Basically a shorthand for console.log(require('util').inspect()).

Rebuilt client asssets and bumped v1 version string.

controllers/uploadController.js

@@ -775,7 +775,7 @@ self.list = async (req, res) => {
     flags: {}
   }
 
-  const orderByObj = {
+  const sortObj = {
     // Cast columns to specific type if they are stored differently
     casts: {
       size: 'integer'
@@ -805,7 +805,7 @@ self.list = async (req, res) => {
     ]
     filterObj.queries = searchQuery.parse(filters, {
       keywords: keywords.concat([
-        'orderby'
+        'sort'
       ]),
       ranges,
       tokenize: true,
@@ -813,16 +813,28 @@ self.list = async (req, res) => {
       offsets: false
     })
 
-    for (const key of keywords)
+    for (const key of keywords) {
-      if (filterObj.queries[key]) {
+      let queryIndex = -1
+      let excludeIndex = -1
+
       // Make sure keyword arrays only contain unique values
+      if (filterObj.queries[key]) {
         filterObj.queries[key] = filterObj.queries[key].filter((v, i, a) => a.indexOf(v) === i)
+        queryIndex = filterObj.queries[key].indexOf('-')
+      }
+      if (filterObj.queries.exclude[key]) {
+        filterObj.queries.exclude[key] = filterObj.queries.exclude[key].filter((v, i, a) => a.indexOf(v) === i)
+        excludeIndex = filterObj.queries.exclude[key].indexOf('-')
+      }
 
       // Flag to match NULL values
-        const index = filterObj.queries[key].indexOf('-')
+      const inQuery = queryIndex !== -1
-        if (index !== -1) {
+      const inExclude = excludeIndex !== -1
-          filterObj.flags[`no${key}`] = true
+      if (inQuery || inExclude) {
-          filterObj.queries[key].splice(index, 1)
+        // Prioritize exclude keys when both types found
+        filterObj.flags[`${key}Null`] = inExclude ? false : inQuery
+        if (inQuery) filterObj.queries[key].splice(queryIndex, 1)
+        if (inExclude) filterObj.queries.exclude[key].splice(excludeIndex, 1)
       }
     }
 
@@ -901,24 +913,24 @@ self.list = async (req, res) => {
       delete filterObj.queries.exclude.user
     }
 
-    // Parse orderby keys
+    // Parse sort keys
-    if (filterObj.queries.orderby) {
+    if (filterObj.queries.sort) {
-      for (const obQuery of filterObj.queries.orderby) {
+      for (const obQuery of filterObj.queries.sort) {
         const tmp = obQuery.toLowerCase().split(':')
 
-        let column = orderByObj.maps[tmp[0]] || tmp[0]
+        let column = sortObj.maps[tmp[0]] || tmp[0]
         let direction = 'asc'
 
-        if (orderByObj.casts[column])
+        if (sortObj.casts[column])
-          column = `cast (\`${column}\` as ${orderByObj.casts[column]})`
+          column = `cast (\`${column}\` as ${sortObj.casts[column]})`
         if (tmp[1] && /^d/.test(tmp[1]))
           direction = 'desc'
 
-        const suffix = orderByObj.nullsLast.includes(column) ? ' nulls last' : ''
+        const suffix = sortObj.nullsLast.includes(column) ? ' nulls last' : ''
-        orderByObj.parsed.push(`${column} ${direction}${suffix}`)
+        sortObj.parsed.push(`${column} ${direction}${suffix}`)
       }
 
-      delete filterObj.queries.orderby
+      delete filterObj.queries.sort
     }
 
     // For some reason, single value won't be in Array even with 'alwaysArray' option
@@ -935,32 +947,50 @@ self.list = async (req, res) => {
       // Sheesh, these look too overbearing...
       this.where(function () {
         // Filter uploads matching any of the supplied 'user' keys and/or NULL flag
-        if (filterObj.uploaders.length)
+        // Prioritze exclude keys when both types found
-          this.orWhereIn('userid', filterObj.uploaders.map(v => v.id))
         if (filterObj.excludeUploaders.length)
           this.orWhereNotIn('userid', filterObj.excludeUploaders.map(v => v.id))
-        if (filterObj.flags.nouser)
+        else if (filterObj.uploaders.length)
+          this.orWhereIn('userid', filterObj.uploaders.map(v => v.id))
+        // Such overbearing logic for NULL values, smh...
+        if ((filterObj.excludeUploaders.length && filterObj.flags.userNull !== false) ||
+          (filterObj.uploaders.length && filterObj.flags.userNull) ||
+          (!filterObj.excludeUploaders.length && !filterObj.uploaders.length && filterObj.flags.userNull))
           this.orWhereNull('userid')
+        else if (filterObj.flags.userNull === false)
+          this.orWhereNotNull('userid')
       }).orWhere(function () {
         // Filter uploads matching any of the supplied 'ip' keys and/or NULL flag
-        if (filterObj.queries.ip)
+        // Same prioritization logics as above
-          this.orWhereIn('ip', filterObj.queries.ip)
         if (filterObj.queries.exclude.ip)
           this.orWhereNotIn('ip', filterObj.queries.exclude.ip)
-        if (filterObj.flags.noip)
+        else if (filterObj.queries.ip)
+          this.orWhereIn('ip', filterObj.queries.ip)
+        // ...
+        if ((filterObj.queries.exclude.ip && filterObj.flags.ipNull !== false) ||
+          (filterObj.queries.ip && filterObj.flags.ipNull) ||
+          (!filterObj.queries.exclude.ip && !filterObj.queries.ip && filterObj.flags.ipNull))
           this.orWhereNull('ip')
+        else if (filterObj.flags.ipNull === false)
+          this.orWhereNotNull('ip')
       }).andWhere(function () {
         // Then, refine using the supplied 'date' and/or 'expiry' ranges
         if (filterObj.queries.date)
+          if (typeof filterObj.queries.date.from === 'number')
             if (typeof filterObj.queries.date.to === 'number')
               this.andWhereBetween('timestamp', [filterObj.queries.date.from, filterObj.queries.date.to])
             else
               this.andWhere('timestamp', '>=', filterObj.queries.date.from)
+          else
+            this.andWhere('timestamp', '<=', filterObj.queries.date.to)
         if (filterObj.queries.expiry)
+          if (typeof filterObj.queries.expiry.from === 'number')
             if (typeof filterObj.queries.expiry.to === 'number')
               this.andWhereBetween('expirydate', [filterObj.queries.expiry.from, filterObj.queries.expiry.to])
             else
               this.andWhere('expirydate', '>=', filterObj.queries.date.from)
+          else
+            this.andWhere('expirydate', '<=', filterObj.queries.date.to)
       }).andWhere(function () {
         // Then, refine using the supplied keywords against their file names
         if (!filterObj.queries.text) return
@@ -975,10 +1005,10 @@ self.list = async (req, res) => {
         if (!filterObj.queries.exclude.text) return
         for (const exclude of filterObj.queries.exclude.text)
           if (exclude.includes('*'))
-            this.orWhere('name', 'not like', exclude.replace(/\*/g, '%'))
+            this.andWhere('name', 'not like', exclude.replace(/\*/g, '%'))
           else
             // If no asterisks, assume partial
-            this.orWhere('name', 'not like', `%${exclude}%`)
+            this.andWhere('name', 'not like', `%${exclude}%`)
       })
   }
 
@@ -1001,12 +1031,12 @@ self.list = async (req, res) => {
     // Only select IPs if we are listing all uploads
     columns.push(all ? 'ip' : 'albumid')
 
-    const orderByRaw = orderByObj.parsed.length
+    const sortRaw = sortObj.parsed.length
-      ? orderByObj.parsed.join(', ')
+      ? sortObj.parsed.join(', ')
       : '`id` desc'
     const files = await db.table('files')
       .where(filter)
-      .orderByRaw(orderByRaw)
+      .orderByRaw(sortRaw)
       .limit(25)
       .offset(25 * offset)
       .select(columns)

logger.js

@@ -45,4 +45,9 @@ self.error = (content, options = {}) => {
   write(content, options)
 }
 
+self.debug = (...args) => {
+  for (const arg of args)
+    console.log(inspect(arg, { depth: Infinity }))
+}
+
 module.exports = self

src/js/dashboard.js

@@ -969,51 +969,57 @@ page.uploadFiltersHelp = element => {
   content.innerHTML = `
     There are 2 filter keys, namely <b>user</b> (username) and <b>ip</b>.
     These keys can be specified more than once.
-    For usernames with whitespaces, wrap them with double quotes (<b>"</b>).
+    For usernames with whitespaces, wrap them with double quotes (<code>"</code>).
+    Special cases such as uploads by non-registered users or have no IPs respectively, use <code>user:-</code> or <code>ip:-</code>.
 
-    There are 2 special filter keys, namely <b>user:-</b> and <b>ip:-</b>, to match uploads by non-registered users and have no IPs respectively.
+    To exclude certain users/ips while still listing every other uploads, add negation sign (<code>-</code>) before the keys.
-
+    Negation sign can also be used to exclude the special cases mentioned above (i.e. <code>-user:-</code> or <code>-ip:-</code>).
-    To exclude certain users/ips while still listing every other uploads, add negation sign (<b>-</b>) before the keys.
-    <b>Unfortunately</b>, this will not show uploads by non-registered users or have no IPs as well, instead you need to also use the special filter keys if you want to include them.
 
     There are 2 range keys: <b>date</b> (upload date) and <b>expiry</b> (expiry date).
-    Their format is: <b>YYYY/MM/DD HH:MM:SS-YYYY/MM/DD HH:MM:SS</b> ("to" date is optional).
+    Their format is: <code>YYYY/MM/DD HH:MM:SS-YYYY/MM/DD HH:MM:SS</code> ("from" date and "to" date respectively).
+    You can specify only one date. If "to" date is missing, 'now' will be used. If "from" date is missing, 'beginning of time' will be used.
     If any of the subsequent date or time units are not specified, their first value will be used (e.g. January for month, 1 for day, and so on).
     If only time is specified, today's date will be used.
-    Meaning the following can be accepted: <b>2020/01/01 01:23</b>, <b>2018/01/01 06</b>, <b>2019/11</b>, <b>12:34:56</b>.
+    In conclusion, the following examples are all valid: <code>date:2020/01/01 01:23-2018/01/01 06</code>, <code>expiry:-2020/05</code>, <code>date:12:34:56</code>.
     These keys can only be specified once each.
 
-    Matches can also be sorted with <b>orderby:columnName[:d[escending]]</b> keys.
+    <b>Timezone?</b> Don't fret, feel free to query the dates with your own timezone!
-    This key require internal column names used in the database (id, userid, and so on), but there are 2 shortcuts, namely <b>date</b> for timestamp column and <b>expiry</b> for expirydate column.
+    API requests to the filter endpoint will attach your browser's timezone offset, so the server will automatically calculate timezone differences.
+
+    Matches can also be sorted with <b>sort</b> keys.
+    Its format is: <code>sort:columnName[:d[escending]]</code>, where <code>:d[escending]</code> is an optional tag to set the direction to descending.
+    This key must be used with internal column names used in the database (<code>id</code>, <code>userid</code>, and so on),
+    but there are 2 shortcuts available: <b>date</b> for <code>timestamp</code> column and <b>expiry</b> for <code>expirydate</code> column.
     This key can also be specified more than once, where their order will decide the sorting steps.
 
-    Any leftover keywords which do not use keys will be matched against the matches' file names.
+    Any leftover keywords which do not use keys (non-keyed keywords) will be matched against the matches' file names.
     Excluding certain keywords is also supported by adding negation sign (<b>-</b>) before the keywords.
 
     <b>Internals:</b>
-    First, it will filter uploads matching ANY of the supplied filter keys AND/OR special filter keys, if any.
+    First, query uploads passing ALL exclusion filter keys OR matching ANY filter keys, if any.
-    Second, it will refine the matches using the supplied <b>date</b> AND/OR <b>expiry</b> range keys, if any.
+    Second, refine matches using range keys, if any.
-    Third, it will refine the matches using the leftover non-keyed keywords, if any.
+    Third, refine matches using ANY non-keyed keywords, if any.
-    Finally, it will sort the matches using the supplied <b>orderby</b> keys, if any.
+    Fourth, filter matches using ALL exclusion non-keyed keywords, if any.
+    Fifth, sort matches using sorting keys, if any.
 
     <b>Examples:</b>
-    Uploads from user named "demo":
-    <code>user:demo</code>
     Uploads from users named "demo" AND/OR "John Doe" AND/OR non-registered users:
     <code>user:demo user:"John Doe" user:-</code>
-    ALL uploads, including from non-registered users, but NOT the ones from user named "demo":
+    ALL uploads, but NOT the ones from user named "demo" AND "John Doe":
-    <code>-user:demo user:-</code>
+    <code>-user:demo -user:"John Doe"</code>
     Uploads from IP "127.0.0.1" AND which file names match "*.rar" OR "*.zip":
     <code>ip:127.0.0.1 *.rar *.zip</code>
     Uploads uploaded since "1 June 2019 00:00:00":
     <code>date:2019/06</code>
-    Uploads uploaded between "7 April 2020 00:00:00" and "7 April 2020 23:59:59":
+    Uploads uploaded between "7 April 2020 12:00:00" and "7 April 2020 23:59:59":
-    <code>date:2020/04/07-2020/04/07 23:59:59</code>
+    <code>date:2020/04/07 12-2020/04/07 23:59:59</code>
+    Uploads uploaded before "5 February 2020 00:00:00":
+    <code>date:-2020/02/05</code>
     Uploads which file names match "*.gz" but NOT "*.tar.gz":
     <code>*.gz -*.tar.gz</code>
     Sort matches by "size" column in ascending and descending order respectively:
-    <code>user:"John Doe" orderby:size</code>
+    <code>user:"John Doe" sort:size</code>
-    <code>*.mp4 user:- orderby:size:d</code>
+    <code>*.mp4 user:- sort:size:d</code>
 
     <b>Friendly reminder:</b> This window can be scrolled up!
   `.trim().replace(/^ {6}/gm, '').replace(/\n/g, '<br>')

src/versions.json

@@ -1,5 +1,5 @@
 {
-  "1": "1587307215",
+  "1": "1587320065",
   "2": "1581416390",
   "3": "1581416390",
   "4": "1581416390",