Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Remove unnecessary double-fetch of the root document

Browse Source
This commit is contained in:
Viktor Lofgren 2024-04-24 14:38:59 +02:00
parent f6db16b313
commit 8b9629f2f6
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
code/processes/crawling-process/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java
View File

@ -117,9 +117,6 @@ public class CrawlerRetreiver implements AutoCloseable {
rootUrl = ok.probedUrl();
}
assert !crawlFrontier.isEmpty();
final SimpleRobotRules robotsRules = fetcher.fetchRobotRules(rootUrl.domain, warcRecorder);
final CrawlDelayTimer delayTimer = new CrawlDelayTimer(robotsRules.getCrawlDelay());
@ -236,6 +233,9 @@ public class CrawlerRetreiver implements AutoCloseable {
catch (Exception ex) {
logger.error("Error configuring link filter", ex);
}
finally {
crawlFrontier.addVisited(rootUrl);
}
}
public HttpFetchResult fetchWriteAndSleep(EdgeUrl top,