Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove annoying log spam in crawler retriever

Browse Source
This commit is contained in:
Viktor Lofgren 2023-06-30 17:08:24 +02:00
parent 8274e8a953
commit 11c26e700e
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java
View File

@ -227,12 +227,12 @@ public class CrawlerRetreiver {
crawlFrontier.addAllToQueue(sitemap);
}
logger.info("Queue is now {}", crawlFrontier.queueSize());
logger.debug("Queue is now {}", crawlFrontier.queueSize());
}
private void sniffRootDocument() {
try {
logger.info("Configuring link filter");
logger.debug("Configuring link filter");
var url = crawlFrontier.peek();
@ -241,6 +241,9 @@ public class CrawlerRetreiver {
return;
var sample = maybeSample.get();
if (sample.documentBody == null)
return;
// Sniff the software based on the sample document
var doc = Jsoup.parse(sample.documentBody.decode());
crawlFrontier.setLinkFilter(linkFilterSelector.selectFilter(doc));