diff --git a/code/processes/live-crawling-process/java/nu/marginalia/livecrawler/SimpleLinkScraper.java b/code/processes/live-crawling-process/java/nu/marginalia/livecrawler/SimpleLinkScraper.java index b1ad42e5..89f5f338 100644 --- a/code/processes/live-crawling-process/java/nu/marginalia/livecrawler/SimpleLinkScraper.java +++ b/code/processes/live-crawling-process/java/nu/marginalia/livecrawler/SimpleLinkScraper.java @@ -92,6 +92,7 @@ public class SimpleLinkScraper implements AutoCloseable { EdgeUrl parsedUrl = optParsedUrl.get(); if (!rules.isAllowed(url)) { + maybeFlagAsBad(parsedUrl); continue; }