Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(link-parser) Make mailing list blocking optional

Browse Source
This commit is contained in:
Viktor Lofgren 2024-10-15 13:48:32 +02:00
parent ab486323f2
commit 89dd201a7b
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/processes/crawling-process/ft-crawl-blocklist/java/nu/marginalia/ip_blocklist/UrlBlocklist.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.ip_blocklist;
import nu.marginalia.model.EdgeUrl;
import nu.marginalia.gregex.GuardedRegexFactory;
import nu.marginalia.model.EdgeUrl;
import java.util.ArrayList;
import java.util.List;
@ -16,6 +16,8 @@ public class UrlBlocklist {
"instagram.com", "youtube.com",
"youtu.be", "amzn.to");
private static final boolean BLOCK_MAILING_LISTS = Boolean.getBoolean("links.block_mailing_lists");
public UrlBlocklist() {
// Don't deep-crawl git repos
patterns.add(s -> s.contains(".git/"));
@ -92,6 +94,10 @@ public class UrlBlocklist {
}
public boolean isMailingListLink(EdgeUrl linkUrl) {
if (!BLOCK_MAILING_LISTS) {
return false;
}
var path = linkUrl.path;
if (path.startsWith("/lists/")) {
return true;