Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(link-parser) Make mailing list blocking optional

Browse Source
This commit is contained in:
Viktor Lofgren 2024-10-15 13:48:32 +02:00
parent ab486323f2
commit 89dd201a7b
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/processes/crawling-process/ft-crawl-blocklist/java/nu/marginalia/ip_blocklist/UrlBlocklist.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.ip_blocklist; package nu.marginalia.ip_blocklist;
import nu.marginalia.model.EdgeUrl;
import nu.marginalia.gregex.GuardedRegexFactory; import nu.marginalia.gregex.GuardedRegexFactory;
import nu.marginalia.model.EdgeUrl;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -16,6 +16,8 @@ public class UrlBlocklist {
"instagram.com", "youtube.com", "instagram.com", "youtube.com",
"youtu.be", "amzn.to"); "youtu.be", "amzn.to");
private static final boolean BLOCK_MAILING_LISTS = Boolean.getBoolean("links.block_mailing_lists");
public UrlBlocklist() { public UrlBlocklist() {
// Don't deep-crawl git repos // Don't deep-crawl git repos
patterns.add(s -> s.contains(".git/")); patterns.add(s -> s.contains(".git/"));
@ -92,6 +94,10 @@ public class UrlBlocklist {
} }
public boolean isMailingListLink(EdgeUrl linkUrl) { public boolean isMailingListLink(EdgeUrl linkUrl) {
if (!BLOCK_MAILING_LISTS) {
return false;
}
var path = linkUrl.path; var path = linkUrl.path;
if (path.startsWith("/lists/")) { if (path.startsWith("/lists/")) {
return true; return true;