Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Reduce log spam from error handling in new sitemap fetcher

Browse Source
This commit is contained in:
Viktor Lofgren 2025-01-20 23:17:13 +01:00
parent 2c67f50a43
commit 579a115243
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
code/processes/crawling-process/java/nu/marginalia/crawl/fetcher/HttpFetcherImpl.java
View File

@ -297,7 +297,7 @@ public class HttpFetcherImpl implements HttpFetcher {
return ret;
}
catch (Exception ex) {
logger.error("Error while fetching sitemaps via " + root, ex);
logger.error("Error while fetching sitemaps via {}: {} ({})", root, ex.getClass().getSimpleName(), ex.getMessage());
return List.of();
}
}
@ -329,6 +329,10 @@ public class HttpFetcherImpl implements HttpFetcher {
}
Document parsedSitemap = Jsoup.parse(parserStream, "UTF-8", sitemapUrl.toString(), Parser.xmlParser());
if (parsedSitemap.childrenSize() == 0) {
return new SitemapResult.SitemapError();
}
String rootTagName = parsedSitemap.child(0).tagName();
return switch (rootTagName.toLowerCase()) {