Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(warc) Fix NPE in WarcRecorder

Browse Source
This commit is contained in:
Viktor Lofgren 2023-12-25 00:54:38 +01:00
parent 1755b646b8
commit c5aab7e8db
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java
View File

@ -289,10 +289,12 @@ public class CrawlerRetreiver implements AutoCloseable {
new ContentType(doc.contentType, "UTF-8"), new ContentType(doc.contentType, "UTF-8"),
doc.documentBody); doc.documentBody);
var parsed = Jsoup.parse(doc.documentBody); if (doc.documentBody != null) {
var parsed = Jsoup.parse(doc.documentBody);
crawlFrontier.enqueueLinksFromDocument(top, parsed); crawlFrontier.enqueueLinksFromDocument(top, parsed);
crawlFrontier.addVisited(top); crawlFrontier.addVisited(top);
}
} }
else if (fetchedDoc instanceof HttpFetchResult.ResultException ex) { else if (fetchedDoc instanceof HttpFetchResult.ResultException ex) {
errorCount ++; errorCount ++;