diff --git a/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java index 56aae5d0..39db4878 100644 --- a/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java +++ b/code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java @@ -381,8 +381,10 @@ public class CrawlerRetreiver implements AutoCloseable { if (docOpt.isPresent()) { var doc = docOpt.get(); - crawlFrontier.enqueueLinksFromDocument(top, doc); - crawlFrontier.addVisited(new EdgeUrl(ok.uri())); + var responseUrl = new EdgeUrl(ok.uri()); + + crawlFrontier.enqueueLinksFromDocument(responseUrl, doc); + crawlFrontier.addVisited(responseUrl); } } else if (fetchedDoc instanceof HttpFetchResult.Result304Raw && reference.doc() != null) {