Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(converter) Fix NPE bugs in converter due to the reintroduction of CrawledDocument.headers

Browse Source
This commit is contained in:
Viktor Lofgren 2024-09-25 12:10:26 +02:00
parent 8b85a58fea
commit e9e8580913
2 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
code/processes/converting-process/java/nu/marginalia/converting/processor/AcceptableAds.java
View File

@ -17,6 +17,9 @@ public class AcceptableAds {
} }
public static boolean hasAcceptableAdsHeader(CrawledDocument document) { public static boolean hasAcceptableAdsHeader(CrawledDocument document) {
if (document.headers != null) {
return document.headers.contains("X-Adblock-Key"); return document.headers.contains("X-Adblock-Key");
} }
return false;
}
} }

12
code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java
View File

@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashSet; import java.util.HashSet;
import java.util.Objects;
import java.util.Set; import java.util.Set;
import static nu.marginalia.converting.model.DisqualifiedException.DisqualificationReason; import static nu.marginalia.converting.model.DisqualifiedException.DisqualificationReason;
@ -127,7 +128,9 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
final EdgeUrl url = new EdgeUrl(crawledDocument.url); final EdgeUrl url = new EdgeUrl(crawledDocument.url);
final var generatorParts = documentGeneratorExtractor.detectGenerator(doc, crawledDocument.headers); final var generatorParts = documentGeneratorExtractor.detectGenerator(doc,
Objects.requireNonNullElse(crawledDocument.headers, "")
);
final var specialization = htmlProcessorSpecializations.select(generatorParts, url); final var specialization = htmlProcessorSpecializations.select(generatorParts, url);
@ -162,7 +165,12 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
throw new DisqualifiedException(DisqualificationReason.QUALITY); throw new DisqualifiedException(DisqualificationReason.QUALITY);
} }
PubDate pubDate = pubDateSniffer.getPubDate(crawledDocument.headers, url, doc, standard, true); PubDate pubDate = pubDateSniffer.getPubDate(
Objects.requireNonNullElse(crawledDocument.headers, ""),
url,
doc,
standard,
true);
EnumSet<DocumentFlags> documentFlags = documentFlags(features, generatorParts.type()); EnumSet<DocumentFlags> documentFlags = documentFlags(features, generatorParts.type());