Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Some small crawler tweaks, plus a test for examining crawler behavior through a simulated server.

Browse Source
This commit is contained in:
vlofgren 2022-09-16 16:59:06 +02:00
parent 5e67391829
commit 123603b0a3
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/QueryParams.java
View File

@ -30,8 +30,17 @@ public class QueryParams {
if (path.endsWith(".cgi")) return true; if (path.endsWith(".cgi")) return true;
if (param.startsWith("id=")) return true; if (param.startsWith("id=")) return true;
if (param.startsWith("p=")) return true; if (param.startsWith("p=")) {
// Don't retain forum links with post-id:s, they're always non-canonical and eat up a lot of
// crawling bandwidth
if (path.endsWith("showthread.php") || path.endsWith("viewtopic.php")) {
return false;
}
return true;
}
if (param.startsWith("i=")) return true; if (param.startsWith("i=")) return true;
if (param.startsWith("start=")) return true;
if (param.startsWith("t=")) return true; if (param.startsWith("t=")) return true;
if (param.startsWith("v=")) return true; if (param.startsWith("v=")) return true;
if (param.startsWith("post=")) return true; if (param.startsWith("post=")) return true;