Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Update URL blocklist

Browse Source 
* Don't crawl MDN mirrors
* More mailing list variants
This commit is contained in:
Viktor Lofgren 2023-07-10 18:04:43 +02:00
parent cbbf60a599
commit fba466d6e2
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
code/features-crawl/crawl-blocklist/src/main/java/nu/marginalia/ip_blocklist/UrlBlocklist.java
View File

@ -71,6 +71,11 @@ public class UrlBlocklist {
return true;
}
// MDN is nice, but we don't need to crawl a bunch of MDN mirrors >.>
if (url.path.contains("developer.mozilla.org")) {
return true;
}
if ("github.com".equals(url.domain.domain)) {
return url.path.chars().filter(c -> c == '/').count() > 2;
}
@ -94,6 +99,12 @@ public class UrlBlocklist {
if (path.contains("mailinglist")) {
return true;
}
if (path.contains("mail-archive")) {
return true;
}
if (path.contains("mailman")) {
return true;
}
return false;
}
}