Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove sitemap related log spam

Browse Source
This commit is contained in:
Viktor Lofgren 2023-06-27 13:59:47 +02:00
parent 7d741ff499
commit d167ad2017
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java
View File

@ -225,8 +225,6 @@ public class CrawlerRetreiver {
checkedSitemaps.add(url.path); checkedSitemaps.add(url.path);
crawlFrontier.addAllToQueue(sitemap); crawlFrontier.addAllToQueue(sitemap);
sitemap.forEach(u -> System.out.println("u" + u));
} }
logger.info("Queue is now {}", crawlFrontier.queueSize()); logger.info("Queue is now {}", crawlFrontier.queueSize());

4
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/fetcher/SitemapRetriever.java
View File

@ -6,6 +6,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.inject.Singleton; import javax.inject.Singleton;
import java.io.FileNotFoundException;
import java.util.*; import java.util.*;
@Singleton @Singleton
@ -20,6 +21,9 @@ public class SitemapRetriever {
try { try {
return sitemapToUrls(parser.parseSiteMap(sitemapUrl.asURL())); return sitemapToUrls(parser.parseSiteMap(sitemapUrl.asURL()));
} }
catch (FileNotFoundException ex) {
return Collections.emptyList();
}
catch (UnknownFormatException ex) { catch (UnknownFormatException ex) {
logger.debug("Unknown sitemap format: {}", sitemapUrl); logger.debug("Unknown sitemap format: {}", sitemapUrl);
return Collections.emptyList(); return Collections.emptyList();