GUI fixes and cleanups (#122)

Co-authored-by: vlofgren <vlofgren@gmail.com> Co-authored-by: vlofgren <vlofgren@marginalia.nu> Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/122
2025-02-23 13:09:00 +00:00 · 2022-10-30 10:08:19 +01:00 · 2022-10-30 10:08:19 +01:00 · e676d8729e
commit e676d8729e
parent c6abbc12f6
34 changed files with 1657 additions and 68 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -0,0 +1,42 @@
+# Contributing
+
+At present this is mostly a solo project, but
+external contributions are very welcome.
+
+This is a bit of a special project, 
+in part because a search engine isn't
+like a text editor that you can just 
+download and tinker with; and in part 
+because it's as much a research project
+as it is a search engine.
+
+If you have an idea for a cool change, 
+send an email to <kontakt@marginalia.nu> and
+we can discuss its feasibility. 
+
+Search is essentially a fractal of interesting
+problems, so even if you don't have an idea, 
+just a skillset (really any), odds are there's
+something interesting I could point you to.
+
+## Release and branches
+
+The search engine has a release cycle of
+once per 6-8 weeks, coinciding with the crawling
+cycle. Where model-breaking changes and changes to
+the crawler can be introduced. 
+
+## Quick Set Up
+
+There is a [Set Up Guide](https://git.marginalia.nu/marginalia/marginalia.nu/wiki/Setup/Search)
+in the wiki. It has a small tendency to oxidize rather
+rapidly since the project currently does not have a
+lot of contributors to test it. If you find a problem
+with the guide, email <kontakt@marginalia.nu>.
+
+## Documentation
+
+What documentation exists resides here:
+
+https://git.marginalia.nu/marginalia/marginalia.nu/wiki
+
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/compiler/DocumentsCompiler.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/compiler/DocumentsCompiler.java
@ -30,7 +30,7 @@ public class DocumentsCompiler {
        var details = doc.details;

        if (details != null) {
-            ret.add(new LoadProcessedDocument(doc.url, doc.state, details.title, details.description, HtmlFeature.encode(details.features), details.standard, details.length, details.hashCode, details.quality));
+            ret.add(new LoadProcessedDocument(doc.url, doc.state, details.title, details.description, HtmlFeature.encode(details.features), details.standard, details.length, details.hashCode, details.quality, details.pubYear));
        }
        else {
            ret.add(new LoadProcessedDocumentWithError(doc.url, doc.state, doc.stateReason));
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/instruction/LoadProcessedDocument.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/interpreter/instruction/LoadProcessedDocument.java
@ -7,6 +7,8 @@ import nu.marginalia.wmsa.edge.model.EdgeUrl;
 import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
 import nu.marginalia.wmsa.edge.model.crawl.EdgeUrlState;

+import javax.annotation.Nullable;
+

 public record LoadProcessedDocument(EdgeUrl url,
                                    EdgeUrlState state,
@ -16,7 +18,8 @@ public record LoadProcessedDocument(EdgeUrl url,
                                    EdgeHtmlStandard standard,
                                    int length,
                                    long hash,
-                                    double quality) implements Instruction
+                                    double quality,
+                                    @Nullable Integer pubYear) implements Instruction
 {
    @Override
    public void apply(Interpreter interpreter) {
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocument.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocument.java
@ -8,6 +8,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import java.sql.SQLException;
+import java.sql.Types;
 import java.util.List;

 import static java.sql.Statement.SUCCESS_NO_INFO;
@ -34,10 +35,11 @@ public class SqlLoadProcessedDocument {
                               IN FEATURES INT,
                               IN STANDARD VARCHAR(32),
                               IN QUALITY DOUBLE,
-                               IN HASH INT)
+                               IN HASH INT,
+                               IN PUB_YEAR SMALLINT)
                       BEGIN
                               SET FOREIGN_KEY_CHECKS=0;
-                               REPLACE INTO EC_PAGE_DATA(ID, TITLE, DESCRIPTION, WORDS_TOTAL, FORMAT, FEATURES, DATA_HASH, QUALITY) VALUES (URL_ID, TITLE, DESCRIPTION, LENGTH, STANDARD, FEATURES, HASH, QUALITY);
+                               REPLACE INTO EC_PAGE_DATA(ID, TITLE, DESCRIPTION, WORDS_TOTAL, FORMAT, FEATURES, DATA_HASH, QUALITY, PUB_YEAR) VALUES (URL_ID, TITLE, DESCRIPTION, LENGTH, STANDARD, FEATURES, HASH, QUALITY, PUB_YEAR);
                               UPDATE EC_URL SET VISITED=1, STATE=STATE WHERE ID=URL_ID;
                               SET FOREIGN_KEY_CHECKS=1;
                       END
@ -62,7 +64,7 @@ public class SqlLoadProcessedDocument {
    public void load(LoaderData data, List<LoadProcessedDocument> documents) {

        try (var conn = dataSource.getConnection();
-             var stmt = conn.prepareCall("CALL INSERT_PAGE_VISIT(?, ?, ?, ?, ?, ?, ?, ?, ?)")) {
+             var stmt = conn.prepareCall("CALL INSERT_PAGE_VISIT(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) {
            conn.setAutoCommit(false);

            int cnt = 0; int batchOffset = 0;
@ -82,6 +84,12 @@ public class SqlLoadProcessedDocument {
                stmt.setString(7, doc.standard().name());
                stmt.setDouble(8, doc.quality());
                stmt.setInt(9, (int) doc.hash());
+                if (doc.pubYear() != null) {
+                    stmt.setShort(10, (short) doc.pubYear().intValue());
+                }
+                else {
+                    stmt.setInt(10, Types.SMALLINT);
+                }
                stmt.addBatch();

                if (++cnt == 100) {
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/model/ProcessedDocumentDetails.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/model/ProcessedDocumentDetails.java
@ -5,6 +5,7 @@ import nu.marginalia.wmsa.edge.converting.processor.logic.HtmlFeature;
 import nu.marginalia.wmsa.edge.model.EdgeUrl;
 import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;

+import javax.annotation.Nullable;
 import java.util.List;
 import java.util.Set;

@ -13,6 +14,9 @@ public class ProcessedDocumentDetails {
    public String title;
    public String description;

+    @Nullable
+    public Integer pubYear;
+
    public int length;
    public double quality;
    public long hashCode;
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/DocumentProcessor.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/DocumentProcessor.java
@ -13,6 +13,8 @@ import nu.marginalia.wmsa.edge.converting.model.DisqualifiedException.Disqualifi
 import nu.marginalia.wmsa.edge.converting.model.ProcessedDocument;
 import nu.marginalia.wmsa.edge.converting.model.ProcessedDocumentDetails;
 import nu.marginalia.wmsa.edge.converting.processor.logic.*;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateSniffer;
 import nu.marginalia.wmsa.edge.crawling.model.CrawledDocument;
 import nu.marginalia.wmsa.edge.crawling.model.CrawledDomain;
 import nu.marginalia.wmsa.edge.crawling.model.CrawlerDocumentStatus;
@ -47,6 +49,7 @@ public class DocumentProcessor {
    private final TitleExtractor titleExtractor;
    private final DocumentKeywordExtractor keywordExtractor;
    private final SummaryExtractor summaryExtractor;
+    private final PubDateSniffer pubDateSniffer;

    private static final DocumentValuator documentValuator = new DocumentValuator();
    private static final LanguageFilter languageFilter = new LanguageFilter();
@ -60,7 +63,8 @@ public class DocumentProcessor {
                             FeatureExtractor featureExtractor,
                             TitleExtractor titleExtractor,
                             DocumentKeywordExtractor keywordExtractor,
-                             SummaryExtractor summaryExtractor)
+                             SummaryExtractor summaryExtractor,
+                             PubDateSniffer pubDateSniffer)
    {
        this.minDocumentLength = minDocumentLength;
        this.minDocumentQuality = minDocumentQuality;
@ -69,6 +73,7 @@ public class DocumentProcessor {
        this.titleExtractor = titleExtractor;
        this.keywordExtractor = keywordExtractor;
        this.summaryExtractor = summaryExtractor;
+        this.pubDateSniffer = pubDateSniffer;
    }

    public ProcessedDocument makeDisqualifiedStub(CrawledDocument crawledDocument) {
@ -177,6 +182,9 @@ public class DocumentProcessor {
        Document doc = Jsoup.parse(crawledDocument.documentBody);

        if (AcceptableAds.hasAcceptableAdsTag(doc)) {
+            // I've never encountered a website where this hasn't been a severe indicator
+            // of spam
+
            throw new DisqualifiedException(DisqualificationReason.ACCEPTABLE_ADS);
        }

@ -204,8 +212,10 @@ public class DocumentProcessor {
        ret.quality = documentValuator.getQuality(crawledDocument, ret.standard, doc, dld);
        ret.hashCode = HashCode.fromString(crawledDocument.documentBodyHash).asLong();

+
        KeywordMetadata keywordMetadata = new KeywordMetadata(ret.quality);

+        PubDate pubDate;
        EdgePageWordSet words;
        if (shouldDoSimpleProcessing(url, ret)) {
            /* Some documents we'll index, but only superficially. This is a compromise
@ -215,17 +225,25 @@ public class DocumentProcessor {
            ret.features = Set.of(HtmlFeature.UNKNOWN);
            words = keywordExtractor.extractKeywordsMinimal(dld, keywordMetadata);
            ret.description = "";
+
+            pubDate = pubDateSniffer.getPubDate(crawledDocument.headers, url, doc, ret.standard, false);
        }
        else {
            ret.features = featureExtractor.getFeatures(crawledDomain, doc, dld);
            words = keywordExtractor.extractKeywords(dld, keywordMetadata);
            ret.description = getDescription(doc);
+
+            pubDate = pubDateSniffer.getPubDate(crawledDocument.headers, url, doc, ret.standard, true);
        }

-        addMetaWords(ret, url, crawledDomain, words);
+        addMetaWords(ret, url, pubDate, crawledDomain, words);

        getLinks(url, ret, doc, words);

+        if (pubDate.hasYear()) {
+            ret.pubYear = pubDate.year();
+        }
+
        return new DetailsWithWords(ret, words);
    }

@ -256,7 +274,7 @@ public class DocumentProcessor {
        return false;
    }

-    private void addMetaWords(ProcessedDocumentDetails ret, EdgeUrl url, CrawledDomain domain, EdgePageWordSet words) {
+    private void addMetaWords(ProcessedDocumentDetails ret, EdgeUrl url, PubDate pubDate, CrawledDomain domain, EdgePageWordSet words) {
        List<String> tagWords = new ArrayList<>();

        var edgeDomain = url.domain;
@ -276,6 +294,13 @@ public class DocumentProcessor {

        ret.features.stream().map(HtmlFeature::getKeyword).forEach(tagWords::add);

+        if (pubDate.year() > 1900) {
+            tagWords.add("year:" + pubDate.year());
+        }
+        if (pubDate.dateIso8601() != null) {
+            tagWords.add("pub:" + pubDate.dateIso8601());
+        }
+
        words.appendWithNoMeta(IndexBlock.Meta, tagWords);
    }

--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/SummaryExtractionFilter.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/SummaryExtractionFilter.java
@ -0,0 +1,250 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.select.NodeFilter;
+
+import java.util.*;
+
+import static org.jsoup.internal.StringUtil.isActuallyWhitespace;
+import static org.jsoup.internal.StringUtil.isInvisibleChar;
+
+public class SummaryExtractionFilter implements NodeFilter {
+
+    public Map<Node, NodeStatistics> statistics = new HashMap<>(10000);
+    public Map<Node, Integer> pos = new HashMap<>(10000);
+    public int cnt = 0;
+
+    @Override
+    public FilterResult head(Node node, int depth) {
+        pos.put(node, cnt++);
+        return FilterResult.CONTINUE;
+    }
+
+    @Override
+    public FilterResult tail(Node node, int depth) {
+        if (node instanceof TextNode tn) {
+            statistics.put(node, new NodeStatistics(tn, 0, textLength(tn.getWholeText()), pos.getOrDefault(tn, cnt)));
+        }
+        else if (node instanceof Element e) {
+            statistics.put(node, aggregateStatistics(e));
+
+            if (shouldPruneTag(e)) {
+                return FilterResult.REMOVE;
+            }
+        }
+
+        return FilterResult.CONTINUE;
+    }
+
+    public boolean shouldPruneTag(Element tag) {
+        String tagName = tag.tagName();
+
+        if ("h1".equalsIgnoreCase(tagName)) return true;
+        if ("h2".equalsIgnoreCase(tagName)) return true;
+        if ("h3".equalsIgnoreCase(tagName)) return true;
+
+        return false;
+    }
+
+    public String getSummary(int maxLength) {
+        List<NodeStatistics> ret = new ArrayList<>(statistics.size());
+        for (var stats : statistics.values()) {
+            if (stats.textToTagRatio() < 0.85) continue;
+            if (!stats.isElement() || !stats.isAppropriateTagType()) continue;
+            if (stats.textLength() < 128) continue;
+            if (stats.isLink()) continue;
+
+            ret.add(stats);
+        }
+        ret.sort(Comparator.comparing(e -> -e.textLength()));
+        if (ret.size() > 32) ret.subList(32, ret.size()).clear();
+        ret.sort(Comparator.comparing(NodeStatistics::pos));
+        if (ret.size() > 3) ret.subList(3, ret.size()).clear();
+        ret.sort(Comparator.comparing(NodeStatistics::isBody));
+        if (ret.size() >= 1) {
+            return StringUtils.abbreviate(ret.get(0).text(), "", maxLength);
+        }
+        return "";
+    }
+
+    private NodeStatistics aggregateStatistics(Element e) {
+        int text = 0;
+        int tag = 0;
+
+        String tagName = e.tagName();
+        if (!tagName.equalsIgnoreCase("br") && !tagName.equalsIgnoreCase("p")) {
+            tag += tagName.length();
+        }
+
+        int numAttributes = e.attributesSize();
+        tag += Math.max(numAttributes - 1, 0);
+
+        if (numAttributes > 0) {
+            var attrs = e.attributes();
+            for (var attr : attrs) {
+                if (Strings.isNullOrEmpty(attr.getValue()))
+                    tag += attr.getKey().length();
+                else {
+                    tag += 3 + attr.getKey().length() + attr.getValue().length();
+                }
+            }
+        }
+
+        for (var childNode : e.childNodes()) {
+            var cn = statistics.get(childNode);
+
+            if (cn != null) {
+                boolean isLink = (tagName.equalsIgnoreCase("a") || cn.isLink());
+                if (isLink) {
+                    tag += cn.tagLength + cn.textLength;
+                }
+                else {
+                    text += cn.textLength;
+                    tag += cn.tagLength;
+                }
+
+                if (!cn.isElement()) {
+                    statistics.remove(cn.node);
+                }
+            }
+        }
+
+        return new NodeStatistics(e, tag, text, pos.getOrDefault(e, cnt));
+    }
+
+    private int textLength(String str) {
+        int length = 0;
+
+        // This is a modified version of JSoup's StringUtil.normaliseWhitespace()
+        // that doesn't do allocation
+
+        int len = str.length();
+        int c;
+        boolean lastWasWhite = false;
+        boolean reachedNonWhite = false;
+
+        for (int i = 0; i < len; i+= Character.charCount(c)) {
+            c = str.codePointAt(i);
+            if (isActuallyWhitespace(c)) {
+                if ((!reachedNonWhite) || lastWasWhite)
+                    continue;
+
+                if (isAscii(c) && Character.isAlphabetic(c)) {
+                    length++;
+                }
+
+                lastWasWhite = true;
+            }
+            else if (!isInvisibleChar(c)) {
+                if (isAscii(c) && Character.isAlphabetic(c)) {
+                    length++;
+                }
+                lastWasWhite = false;
+                reachedNonWhite = true;
+            }
+        }
+
+        return length;
+    }
+
+    public boolean isAscii(int cp) {
+        return (cp & ~0x7F) == 0;
+    }
+
+    public record NodeStatistics(Node node, int tagLength, int textLength, int pos) {
+        public double textToTagRatio() {
+            if (textLength == 0) return 1;
+
+            return textLength / (double)(tagLength + textLength);
+        }
+
+        public String text() {
+            if (node instanceof Element e) {
+                return e.text();
+            }
+            else if (node instanceof TextNode tn) {
+                return tn.text();
+            }
+            return "";
+        }
+
+        public boolean isElement() {
+            return node instanceof Element;
+        }
+
+        public boolean isLink() {
+            if (node instanceof Element el) {
+                return "a".equalsIgnoreCase(el.tagName());
+            }
+            return false;
+        }
+
+        public boolean isAppropriateTagType() {
+
+            if (node instanceof Element el) {
+                String tagName = el.tagName();
+                if ("blockquote".equalsIgnoreCase(tagName))
+                    return false;
+                if ("tt".equalsIgnoreCase(tagName))
+                    return false;
+                if ("ol".equalsIgnoreCase(tagName))
+                    return false;
+                if ("ul".equalsIgnoreCase(tagName))
+                    return false;
+                if ("li".equalsIgnoreCase(tagName))
+                    return false;
+                if ("h1".equalsIgnoreCase(tagName))
+                    return false;
+                if ("h2".equalsIgnoreCase(tagName))
+                    return false;
+                if ("h3".equalsIgnoreCase(tagName))
+                    return false;
+                if ("th".equalsIgnoreCase(tagName))
+                    return false;
+                if ("td".equalsIgnoreCase(tagName))
+                    return false;
+                if ("tbody".equalsIgnoreCase(tagName))
+                    return false;
+                if ("html".equalsIgnoreCase(tagName))
+                    return false;
+                if ("title".equalsIgnoreCase(tagName))
+                    return false;
+                if ("#root".equalsIgnoreCase(tagName))
+                    return false;
+            }
+
+            if (node.parent() instanceof Element elp) {
+                if ("a".equals(elp.tagName()))
+                    return false;
+            }
+
+            return true;
+        }
+
+        public boolean isBody() {
+            if (node instanceof Element el) {
+                return "body".equalsIgnoreCase(el.tagName());
+            }
+            return false;
+        }
+
+        public String tagName() {
+            if (node instanceof Element el) {
+                return el.tagName();
+            }
+            return '$'+node.getClass().getSimpleName();
+        }
+
+        public String toString() {
+            return String.format("NodeStatistics[%s %d p %d %d]", tagName(), pos, tagLength, textLength);
+        }
+
+        public double sortValue() {
+            return -textToTagRatio() * Math.log(1 + textLength) / Math.log(1+pos);
+        }
+    }
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/SummaryExtractor.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/SummaryExtractor.java
@ -11,7 +11,7 @@ import java.util.regex.Pattern;
 public class SummaryExtractor {
    private final int maxSummaryLength;

-    private final Pattern truncatedCharacters = Pattern.compile("[^a-zA-Z0-9.,!?\\-'\"]+|[\\-.,!?' ]{3,}");
+    private final Pattern truncatedCharacters = Pattern.compile("[\\-.,!?' ]{3,}");

    @Inject
    public SummaryExtractor(@Named("max-summary-length") Integer maxSummaryLength) {
@ -19,12 +19,9 @@ public class SummaryExtractor {
    }

    public String extractSummary(Document parsed) {
-        var cleanDoc = parsed.clone();
-        cleanDoc.select("header,nav,#header,#nav,#navigation,.header,.nav,.navigation,ul,li").remove();
-
        String summaryString;

-        summaryString = extractSummaryRaw(cleanDoc);
+        summaryString = extractSummaryRaw(parsed);
        summaryString = truncatedCharacters.matcher(summaryString).replaceAll(" ");
        summaryString = StringUtils.abbreviate(summaryString, "", maxSummaryLength);

@ -36,9 +33,13 @@ public class SummaryExtractor {

        String maybe;

-        // Plan A
+        parsed.select("header,nav,#header,#nav,#navigation,.header,.nav,.navigation,ul,li").remove();

-        maybe = getSummaryByTagDensity(parsed);
+        // Plan A
+        maybe = getSummaryNew(parsed.clone());
+        if (!maybe.isBlank()) return maybe;
+
+        maybe = getSummaryByTagDensity(parsed.clone());
        if (!maybe.isBlank()) return maybe;

        // Plan B: Open Graph Description
@ -53,6 +54,14 @@ public class SummaryExtractor {
        return lastDitchSummaryEffort(parsed);
    }

+    private String getSummaryNew(Document parsed) {
+        var filter = new SummaryExtractionFilter();
+
+        parsed.filter(filter);
+
+        return filter.getSummary(maxSummaryLength+32);
+    }
+
    private String getSummaryByTagDensity(Document parsed) {
        StringBuilder content = new StringBuilder();

@ -92,6 +101,7 @@ public class SummaryExtractor {

        return parsed.body().text();
    }
+
    private double htmlTagDensity(Element elem) {
        return (double) elem.text().length() / elem.html().length();
    }
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDate.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDate.java
@ -0,0 +1,46 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+
+public record PubDate(String dateIso8601, int year) {
+
+    // First year we'll believe something can have been published on the web
+    // cut off at 1995 to reduce false positive error rate; number of bona fide
+    // documents from these years are so few almost all hits are wrong
+    
+    public static final int MIN_YEAR = 1995;
+
+    // Last year we'll believe something can be published in
+    public static final int MAX_YEAR = LocalDate.now().getYear() + 1;
+
+
+    public PubDate() {
+        this(null, Integer.MIN_VALUE);
+    }
+
+    public PubDate(LocalDate date) {
+        this(date.format(DateTimeFormatter.ISO_DATE), date.getYear());
+    }
+
+    public boolean isEmpty() {
+        return year == Integer.MIN_VALUE;
+    }
+
+    public String describe() {
+        if (dateIso8601 != null)
+            return dateIso8601;
+
+        if (hasYear())
+            return Integer.toString(year);
+
+        return "";
+    }
+
+    public static boolean isValidYear(int year) {
+        return year >= MIN_YEAR && year <= MAX_YEAR;
+    }
+    public boolean hasYear() {
+        return isValidYear(this.year);
+    }
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateEffortLevel.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateEffortLevel.java
@ -0,0 +1,6 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate;
+
+public enum PubDateEffortLevel {
+    LOW,
+    HIGH
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateHeuristic.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateHeuristic.java
@ -0,0 +1,12 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate;
+
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public interface PubDateHeuristic {
+
+    Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard);
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateParser.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateParser.java
@ -0,0 +1,178 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate;
+
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+
+import java.time.DateTimeException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.regex.Pattern;
+
+public class PubDateParser {
+
+    public static Optional<PubDate> attemptParseDate(String date) {
+        return Optional.ofNullable(date)
+                .filter(str -> str.length() >= 4 && str.length() < 32)
+                .flatMap(str ->
+                        parse8601(str)
+                                .or(() -> parse1123(str))
+                                .or(() -> dateFromHighestYearLookingSubstring(str))
+                )
+                .filter(PubDateParser::validateDate);
+    }
+
+    public static OptionalInt parseYearString(String yearString) {
+        try {
+            return OptionalInt.of(Integer.parseInt(yearString));
+        }
+        catch (NumberFormatException ex) {
+            return OptionalInt.empty();
+        }
+    }
+
+
+    private static final Pattern yearPattern = Pattern.compile("\\d{4}");
+
+    public static Optional<PubDate> dateFromHighestYearLookingSubstring(String maybe) {
+        var matcher = yearPattern.matcher(maybe);
+
+        int min = PubDate.MAX_YEAR + 1;
+        int max = PubDate.MIN_YEAR - 1;
+
+        for (int i = 0; i < maybe.length() && matcher.find(i); i = matcher.end()) {
+
+            String segment = maybe.substring(matcher.start(), matcher.end());
+            OptionalInt year = parseYearString(segment);
+
+            if (year.isEmpty())
+                continue;
+
+            int y = year.getAsInt();
+            if (PubDate.isValidYear(y)) {
+                if (max < y) max = y;
+                if (min > y) min = y;
+            }
+        }
+
+        if (max != min && PubDate.isValidYear(min) && PubDate.isValidYear(max)) {
+            return Optional.of(new PubDate(null, guessYear(min, max)));
+        }
+
+        if (max >= PubDate.MIN_YEAR)
+            return Optional.of(new PubDate(null, max));
+        else
+            return Optional.empty();
+    }
+
+
+    public static Optional<PubDate> dateFromHighestYearLookingSubstringWithGuess(String maybe, int guess) {
+        var matcher = yearPattern.matcher(maybe);
+
+        int min = PubDate.MAX_YEAR + 1;
+        int max = PubDate.MIN_YEAR - 1;
+
+        for (int i = 0; i < maybe.length() && matcher.find(i); i = matcher.end()) {
+
+            String segment = maybe.substring(matcher.start(), matcher.end());
+            OptionalInt year = parseYearString(segment);
+
+            if (year.isEmpty())
+                continue;
+
+            int y = year.getAsInt();
+            if (PubDate.isValidYear(y)) {
+                if (max < y) max = y;
+                if (min > y) min = y;
+            }
+        }
+
+        if (max != min && PubDate.isValidYear(min) && PubDate.isValidYear(max)) {
+            return Optional.of(new PubDate(null, guessYear(min, max, guess)));
+        }
+
+        if (max >= PubDate.MIN_YEAR)
+            return Optional.of(new PubDate(null, max));
+        else
+            return Optional.empty();
+    }
+
+    public static int guessYear(int min, int max, int educatedGuess) {
+        int var = max - min;
+
+        if (var < 3)
+            return min;
+
+        int avg = (max + min) / 2;
+        int guess = (avg + educatedGuess) / 2;
+
+        if (guess < min)
+            return min;
+        if (guess > max)
+            return max;
+
+        return guess;
+    }
+
+    public static int guessYear(int min, int max) {
+        return (max + min) / 2;
+    }
+
+    public static int guessYear(EdgeHtmlStandard standard) {
+        // Create some jitter to avoid having documents piling up in the same four years
+        // as this would make searching in those years disproportionately useless
+
+        double guess = standard.yearGuess + ThreadLocalRandom.current().nextGaussian();
+
+        if (guess < PubDate.MIN_YEAR) {
+            return PubDate.MIN_YEAR;
+        }
+        if (guess > PubDate.MAX_YEAR) {
+            return PubDate.MAX_YEAR;
+        }
+        return (int) guess;
+    }
+
+    public static Optional<PubDate> parse8601(String maybe) {
+        return parseOptionally(maybe, DateTimeFormatter.ISO_DATE)
+                .or(() -> parseOptionallyWithTime(maybe, DateTimeFormatter.ISO_DATE_TIME))
+                .or(() -> parseOptionallyWithZonedTime(maybe, DateTimeFormatter.ISO_DATE_TIME))
+                .map(PubDate::new);
+    }
+
+    public static Optional<PubDate> parse1123(String maybe) {
+        return parseOptionally(maybe, DateTimeFormatter.RFC_1123_DATE_TIME)
+                .map(PubDate::new);
+    }
+
+    public static Optional<LocalDate> parseOptionally(String str, DateTimeFormatter formatter) {
+        try {
+            return Optional.of(LocalDate.parse(str, formatter));
+        }
+        catch (DateTimeException ex) {
+            return Optional.empty();
+        }
+    }
+    public static Optional<LocalDate> parseOptionallyWithTime(String str, DateTimeFormatter formatter) {
+        try {
+            return Optional.of(LocalDateTime.parse(str, formatter).toLocalDate());
+        }
+        catch (DateTimeException ex) {
+            return Optional.empty();
+        }
+    }
+    public static Optional<LocalDate> parseOptionallyWithZonedTime(String str, DateTimeFormatter formatter) {
+        try {
+            return Optional.of(ZonedDateTime.parse(str, formatter).toLocalDate());
+        }
+        catch (DateTimeException ex) {
+            return Optional.empty();
+        }
+    }
+    public static boolean validateDate(PubDate date) {
+        return (date.year() >= PubDate.MIN_YEAR && date.year() <= PubDate.MAX_YEAR);
+    }
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateSniffer.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/PubDateSniffer.java
@ -0,0 +1,50 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic.*;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PubDateSniffer {
+
+    private final List<PubDateHeuristic> heuristics = new ArrayList<>();
+
+    public PubDateSniffer() {
+        heuristics.add(new PubDateHeuristicHtml5ItempropDateTag());
+        heuristics.add(new PubDateHeuristicHtml5ArticleDateTag());
+        heuristics.add(new PubDateHeuristicJSONLD());
+        heuristics.add(new PubDateHeuristicMicrodata());
+        heuristics.add(new PubDateHeuristicOpenGraph());
+        heuristics.add(new PubDateHeuristicRDFaTag());
+
+        // The more questionable heuristics should be kept below this line
+        heuristics.add(new PubDateHeuristicUrlPatternPass1());
+
+        heuristics.add(new PubDateHeuristicDOMParsingPass1());
+        heuristics.add(new PubDateHeuristicHtml5AnyTimeTag());
+
+        heuristics.add(new PubDateHeuristicDOMParsingPass2());
+        heuristics.add(new PubDateHeuristicUrlPatternPass2());
+
+        heuristics.add(new PubDateHeuristicLastModified());
+        // This is complete guesswork
+
+        heuristics.add(new PubDateHeuristicGuessFromHtmlStandard());
+    }
+
+    public PubDate getPubDate(String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard, boolean runExpensive) {
+        final PubDateEffortLevel effortLevel = runExpensive ? PubDateEffortLevel.HIGH : PubDateEffortLevel.LOW;
+
+        for (var heuristic : heuristics) {
+            var maybe = heuristic.apply(effortLevel, headers, url, document, htmlStandard);
+            if (maybe.isPresent())
+                return maybe.get();
+        }
+
+        return new PubDate();
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java
@ -0,0 +1,148 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jetbrains.annotations.NotNull;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.select.NodeFilter;
+
+import java.util.Optional;
+
+public class PubDateHeuristicDOMParsingPass1 implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        if (effortLevel == PubDateEffortLevel.LOW)
+            return Optional.empty();
+
+        DateExtractingNodeVisitorPass filter = new DateExtractingNodeVisitorPass(htmlStandard);
+
+        document.filter(filter);
+
+        return Optional.ofNullable(filter.pubDate);
+    }
+
+
+    private static class DateExtractingNodeVisitorPass implements NodeFilter {
+        public PubDate pubDate;
+        private final EdgeHtmlStandard htmlStandard;
+
+        private DateExtractingNodeVisitorPass(EdgeHtmlStandard htmlStandard) {
+            this.htmlStandard = htmlStandard;
+        }
+
+        @NotNull
+        @Override
+        public FilterResult head(@NotNull Node node, int depth) {
+            if (node instanceof TextNode tn) onTextNode(tn);
+            if (node instanceof Element el) onElementNode(el);
+
+            if (hasPubDate()) {
+                return FilterResult.STOP;
+            }
+            return FilterResult.CONTINUE;
+        }
+
+        public void onTextNode(TextNode tn) {
+            String text = tn.getWholeText();
+
+            if (text.length() < 32 && isCandidatForCopyrightNotice(text)) {
+                parse(text);
+            }
+        }
+
+
+        public void onElementNode(Element el) {
+            if (hasCommonClass(el)) {
+                parse(el.text());
+            }
+
+            if (!hasPubDate())
+                tryParsePhpBBDate(el);
+        }
+
+
+        public boolean isCandidatForCopyrightNotice(String text) {
+            if (text.contains("ublished"))
+                return true;
+            if (text.contains("opyright"))
+                return true;
+            if (text.contains("&copy;"))
+                return true;
+            if (text.contains("(c)"))
+                return true;
+
+            return false;
+        }
+
+
+        public boolean hasCommonClass(Element el) {
+            var classes = el.classNames();
+
+            return classes.contains("entry-meta") // wordpress
+                    || classes.contains("byline")
+                    || classes.contains("author")
+                    || classes.contains("submitted")
+                    || el.id().contains("footer-info-lastmod"); // mediawiki
+        }
+
+        public void tryParsePhpBBDate(Element el) {
+
+            /* Match HTML on the form <div>[...] <b>Posted:</b> Sun Oct 03, 2010 5:37 pm&nbsp;</div>
+             * this is used on old phpBB message boards
+             *
+             * Schematically the DOM looks like this
+             *
+             *              b - TextNode[ Sun Oct 03, 2010 5:37 pm&nbsp;]
+             *              |
+             *          TextNode[Posted:]
+             */
+            if ("b".equals(el.tagName())
+                        && el.childNodeSize() == 1
+                        && el.childNode(0) instanceof TextNode ctn
+                        && "Posted:".equals(ctn.getWholeText())
+                        && el.nextSibling() instanceof TextNode ntn
+                )
+            {
+                parse(ntn.getWholeText());
+            }
+        }
+
+
+        public boolean hasPubDate() {
+            return pubDate != null;
+        }
+        public void setPubDate(PubDate pubDate) {
+            this.pubDate = pubDate;
+        }
+
+        @NotNull
+        @Override
+        public FilterResult tail(@NotNull Node node, int depth) {
+            return FilterResult.CONTINUE;
+        }
+
+        private void parse(String text) {
+            if (htmlStandard == EdgeHtmlStandard.UNKNOWN) {
+                PubDateParser
+                        .dateFromHighestYearLookingSubstring(text)
+                        .ifPresent(this::setPubDate);
+            }
+            else {
+                PubDateParser
+                        .dateFromHighestYearLookingSubstringWithGuess(text, htmlStandard.yearGuess)
+                        .ifPresent(this::setPubDate);
+            }
+        }
+
+
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java
@ -0,0 +1,123 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jetbrains.annotations.NotNull;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.select.NodeFilter;
+
+import java.util.Optional;
+
+public class PubDateHeuristicDOMParsingPass2 implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        if (effortLevel == PubDateEffortLevel.LOW)
+            return Optional.empty();
+
+        DateExtractingNodeVisitor filter = new DateExtractingNodeVisitor(htmlStandard);
+
+        document.filter(filter);
+
+        return Optional.ofNullable(filter.pubDate);
+    }
+
+
+    private static class DateExtractingNodeVisitor implements NodeFilter {
+        public PubDate pubDate;
+        private final EdgeHtmlStandard htmlStandard;
+
+        private DateExtractingNodeVisitor(EdgeHtmlStandard htmlStandard) {
+            this.htmlStandard = htmlStandard;
+        }
+
+        @NotNull
+        @Override
+        public FilterResult head(@NotNull Node node, int depth) {
+            if (node instanceof TextNode tn) onTextNode(tn);
+
+            if (hasPubDate()) {
+                return FilterResult.STOP;
+            }
+            return FilterResult.CONTINUE;
+        }
+
+        public void onTextNode(TextNode tn) {
+            String text = tn.getWholeText();
+
+            if (isPossibleCandidate(text)) {
+                parse(text);
+            }
+        }
+
+
+        public boolean hasPubDate() {
+            return pubDate != null;
+        }
+        public void setPubDate(PubDate pubDate) {
+            this.pubDate = pubDate;
+        }
+
+        @NotNull
+        @Override
+        public FilterResult tail(@NotNull Node node, int depth) {
+            return FilterResult.CONTINUE;
+        }
+
+        private void parse(String text) {
+            if (htmlStandard == EdgeHtmlStandard.UNKNOWN) {
+                PubDateParser
+                        .dateFromHighestYearLookingSubstring(text)
+                        .ifPresent(this::setPubDate);
+            }
+            else {
+                PubDateParser
+                        .dateFromHighestYearLookingSubstringWithGuess(text, htmlStandard.yearGuess)
+                        .ifPresent(this::setPubDate);
+            }
+        }
+
+
+    }
+
+    // This is basically the regex (^|[ ./\-])(\d{4})([ ./\-]$), but
+    // unchecked regexes are too slow
+
+    public static boolean isPossibleCandidate(String text) {
+        if (text.length() >= 4 && text.length() < 24) {
+            int ct = 0;
+            char prevC = ' ';
+            boolean goodStart = true;
+            for (int i = 0; i < text.length(); i++) {
+                char c = text.charAt(i);
+                if (Character.isDigit(c)) {
+                    if (ct++ == 0) {
+                        goodStart = isGoodBreak(prevC);
+                    }
+                }
+                else {
+                    if (ct == 4 && goodStart && isGoodBreak(c)) return true;
+                    else {
+                        ct = 0;
+                    }
+                }
+                prevC = c;
+            }
+
+            if (ct == 4 && goodStart)
+                return true;
+        }
+        return false;
+    }
+
+    private static boolean isGoodBreak(char c) {
+        return "./-,".indexOf(c) >= 0 || Character.isSpaceChar(c);
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java
@ -0,0 +1,23 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicGuessFromHtmlStandard implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        if (htmlStandard == EdgeHtmlStandard.UNKNOWN)
+            return Optional.empty();
+
+        return Optional.of(new PubDate(null, PubDateParser.guessYear(htmlStandard)));
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java
@ -0,0 +1,33 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicHtml5AnyTimeTag implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        // HTML5, alternative approach
+        for (var tag : document.select("time")) {
+            var maybeDate = PubDateParser.attemptParseDate(tag.attr("datetime"));
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+
+            maybeDate = PubDateParser.attemptParseDate(tag.wholeText());
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java
@ -0,0 +1,28 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicHtml5ArticleDateTag implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        // HTML5
+        for (var tag : document.select("time[pubdate=\"pubdate\"]")) {
+            var maybeDate = PubDateParser.attemptParseDate(tag.attr("datetime"));
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java
@ -0,0 +1,27 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicHtml5ItempropDateTag implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        for (var tag : document.select("time[itemprop=\"datePublished\"]")) {
+            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicJSONLD.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicJSONLD.java
@ -0,0 +1,49 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonSyntaxException;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicJSONLD implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        for (var tag : document.select("script[type=\"application/ld+json\"]")) {
+            var maybeDate = parseLdJson(tag.data())
+                    .flatMap(PubDateParser::attemptParseDate);
+
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+
+    private static class JsonModel {
+        String datePublished;
+    }
+    private static Gson gson = new GsonBuilder().create();
+
+    public Optional<String> parseLdJson(String content) {
+        try {
+            var model = gson.fromJson(content, JsonModel.class);
+            return Optional.ofNullable(model)
+                    .map(m -> m.datePublished);
+        }
+        catch (JsonSyntaxException ex) {
+            return Optional.empty();
+        }
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicLastModified.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicLastModified.java
@ -0,0 +1,29 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicLastModified implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        String lmString = "last-modified: ";
+        int offset = headers.toLowerCase().indexOf(lmString);
+
+        if (offset < 0)
+            return Optional.empty();
+        int end = headers.indexOf('\n', offset);
+        if (end < 0) end = headers.length();
+
+        String lmDate = headers.substring(offset + lmString.length(), end);
+        return PubDateParser.attemptParseDate(lmDate);
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicMicrodata.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicMicrodata.java
@ -0,0 +1,28 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicMicrodata implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+
+        for (var tag : document.select("meta[itemprop=\"datePublished\"]")) {
+            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicOpenGraph.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicOpenGraph.java
@ -0,0 +1,27 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicOpenGraph implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        // OG
+        for (var tag : document.select("meta[property=\"article:published_time\"]")) {
+            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+        }
+
+        return Optional.empty();
+    }
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicRDFaTag.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicRDFaTag.java
@ -0,0 +1,27 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+
+public class PubDateHeuristicRDFaTag implements PubDateHeuristic {
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        for (var tag : document.select("meta[property=\"datePublished\"]")) {
+            var maybeDate = PubDateParser.attemptParseDate(tag.attr("content"));
+            if (maybeDate.isPresent()) {
+                return maybeDate;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java
@ -0,0 +1,45 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.regex.Pattern;
+
+public class PubDateHeuristicUrlPatternPass1 implements PubDateHeuristic {
+
+    private static final Pattern yearUrlPattern = Pattern.compile("/\\d{4}/");
+
+    // False positive rate is much higher in the 1990s, only include 2000s+ in pass 1
+    private static final int MIN_URL_PATTERN_YEAR = 2000;
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        final String urlString = url.path;
+
+        var matcher = yearUrlPattern.matcher(urlString);
+
+        for (int i = 0; i < urlString.length() && matcher.find(i); i = matcher.end()) {
+
+            String segment = urlString.substring(matcher.start() + 1, matcher.end() - 1);
+
+            OptionalInt year = PubDateParser.parseYearString(segment);
+
+            if (year.isEmpty())
+                continue;
+
+            int y = year.getAsInt();
+            if (y >= MIN_URL_PATTERN_YEAR && y <= PubDate.MAX_YEAR) {
+                return Optional.of(new PubDate(null, y));
+            }
+        }
+
+        return Optional.empty();
+    }
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/processor/logic/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java
@ -0,0 +1,42 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDate;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateEffortLevel;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateHeuristic;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.nodes.Document;
+
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.regex.Pattern;
+
+public class PubDateHeuristicUrlPatternPass2 implements PubDateHeuristic {
+
+    private static final Pattern yearUrlPattern = Pattern.compile("/\\d{4}/");
+
+    @Override
+    public Optional<PubDate> apply(PubDateEffortLevel effortLevel, String headers, EdgeUrl url, Document document, EdgeHtmlStandard htmlStandard) {
+        final String urlString = url.path;
+
+        var matcher = yearUrlPattern.matcher(urlString);
+
+        for (int i = 0; i < urlString.length() && matcher.find(i); i = matcher.end()) {
+
+            String segment = urlString.substring(matcher.start() + 1, matcher.end() - 1);
+
+            OptionalInt year = PubDateParser.parseYearString(segment);
+
+            if (year.isEmpty())
+                continue;
+
+            int y = year.getAsInt();
+            if (y >= PubDate.MIN_YEAR && y <= PubDate.MAX_YEAR) {
+                return Optional.of(new PubDate(null, y));
+            }
+        }
+
+        return Optional.empty();
+    }
+}
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/explorer/ExplorerService.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/explorer/ExplorerService.java
@ -117,12 +117,12 @@ public class ExplorerService extends Service {
                    NV.NEIGHBOR_NAME,
                    NV.RELATEDNESS,
                    (LV.DOMAIN_ID IS NOT NULL),
-                    (STATE = 'ACTIVE' OR STATE='SOCIAL_MEDIA'),
+                    (STATE = 'ACTIVE' OR STATE='SOCIAL_MEDIA' OR STATE='REDIR'),
                    INDEXED > 0
                FROM EC_NEIGHBORS_VIEW NV
                LEFT JOIN EC_NEIGHBORS_VIEW LV ON (NV.NEIGHBOR_ID=LV.DOMAIN_ID)
                INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=NV.NEIGHBOR_ID
-                WHERE NV.DOMAIN_ID=?
+                WHERE NV.DOMAIN_ID IN (?,?)
                GROUP BY NV.NEIGHBOR_ID
                ORDER BY NV.RELATEDNESS DESC
                """);
@ -131,12 +131,12 @@ public class ExplorerService extends Service {
                    NV.DOMAIN_NAME,
                    NV.RELATEDNESS,
                    (LV.NEIGHBOR_ID IS NOT NULL),
-                    (STATE = 'ACTIVE' OR STATE='SOCIAL_MEDIA'),
+                    (STATE = 'ACTIVE' OR STATE='SOCIAL_MEDIA' OR STATE='REDIR'),
                    INDEXED > 0
                FROM EC_NEIGHBORS_VIEW NV
                LEFT JOIN EC_NEIGHBORS_VIEW LV ON (NV.DOMAIN_ID=LV.NEIGHBOR_ID)
                INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=NV.DOMAIN_ID
-                WHERE NV.NEIGHBOR_ID=?
+                WHERE NV.NEIGHBOR_ID IN (?,?)
                GROUP BY NV.DOMAIN_ID
                ORDER BY NV.RELATEDNESS DESC
                """
@ -145,6 +145,8 @@ public class ExplorerService extends Service {
             ) {

            stmt.setInt(1, domainIdInformation.domainId);
+            stmt.setInt(2, domainIdInformation.aliasId);
+
            var rsp = stmt.executeQuery();
            while (rsp.next()) {

@ -172,6 +174,8 @@ public class ExplorerService extends Service {
            }

            stmtRev.setInt(1, domainIdInformation.domainId);
+            stmtRev.setInt(2, domainIdInformation.aliasId);
+
            rsp = stmtRev.executeQuery();
            while (rsp.next()) {

@ -211,22 +215,24 @@ public class ExplorerService extends Service {

        try (var conn = dataSource.getConnection();
             var stmt = conn.prepareStatement("""
-                SELECT IFNULL(ALIAS.ID, DOMAIN.ID), DOMAIN.INDEXED>0 OR ALIAS.INDEXED>0, ALIAS.DOMAIN_NAME 
+                SELECT DOMAIN.ID, IFNULL(ALIAS.ID, DOMAIN.ID), DOMAIN.INDEXED>0 OR ALIAS.INDEXED>0, ALIAS.DOMAIN_NAME 
                FROM EC_DOMAIN DOMAIN 
-                LEFT JOIN EC_DOMAIN ALIAS ON DOMAIN.DOMAIN_ALIAS=ALIAS.ID                
+                LEFT JOIN EC_DOMAIN ALIAS ON DOMAIN.DOMAIN_ALIAS=ALIAS.ID
                WHERE DOMAIN.DOMAIN_NAME=?
                """)) {
+
            stmt.setString(1, query);
            var rsp = stmt.executeQuery();
            if (rsp.next()) {
                return new DomainIdInformation(
                        rsp.getInt(1),
-                        rsp.getBoolean(2),
-                        rsp.getString(3)
+                        rsp.getInt(2),
+                        rsp.getBoolean(3),
+                        rsp.getString(4)
                );
            }
        }
-        return new DomainIdInformation(-1,  false, null);
+        return new DomainIdInformation(-1, -1,  false, null);
    }

    private String trimUrlJunk(String query) {
@ -245,7 +251,7 @@ public class ExplorerService extends Service {
        return query;
    }

-    record DomainIdInformation(int domainId, boolean indexed, String alias) {
+    record DomainIdInformation(int domainId, int aliasId, boolean indexed, String alias) {
        boolean isPresent() {
            return domainId >= 0;
        }
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/crawl/EdgeHtmlStandard.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/model/crawl/EdgeHtmlStandard.java
@ -1,19 +1,22 @@
 package nu.marginalia.wmsa.edge.model.crawl;

 public enum EdgeHtmlStandard {
-    PLAIN(0, 1),
-    UNKNOWN(0, 1),
-    HTML123(0, 1),
-    HTML4(-0.1, 1.05),
-    XHTML(-0.1, 1.05),
-    HTML5(0.5, 1.1);
+    PLAIN(0, 1, 1993),
+    UNKNOWN(0, 1, 2000),
+    HTML123(0, 1, 1997),
+    HTML4(-0.1, 1.05, 2006),
+    XHTML(-0.1, 1.05, 2006),
+    HTML5(0.5, 1.1, 2018);

    public final double offset;
    public final double scale;

-    EdgeHtmlStandard(double offset, double scale) {
+    public final int yearGuess;
+
+    EdgeHtmlStandard(double offset, double scale, int yearGuess) {
        this.offset = offset;
        this.scale = scale;
+        this.yearGuess = yearGuess;
    }

 }
--- a/marginalia_nu/src/main/resources/sql/edge-crawler-cache.sql
+++ b/marginalia_nu/src/main/resources/sql/edge-crawler-cache.sql
@ -77,6 +77,8 @@ CREATE TABLE IF NOT EXISTS EC_PAGE_DATA (
    DATA_HASH INTEGER NOT NULL,
    QUALITY DOUBLE NOT NULL,

+    PUB_YEAR SMALLINT,
+
    FOREIGN KEY (ID) REFERENCES EC_URL(ID) ON DELETE CASCADE
 )
 CHARACTER SET utf8mb4
--- a/marginalia_nu/src/main/resources/templates/edge/index.hdb
+++ b/marginalia_nu/src/main/resources/templates/edge/index.hdb
@ -28,38 +28,28 @@
        <h2>Publicity, Discussion and Events</h2>
        <div class="info">
            <dl>
-                <dt><a href="https://www.hs.fi/visio/art-2000009139237.html">Google ei en&auml;&auml; tide&auml;</a></dt>
+                <dt><a href="https://memex.marginalia.nu/log/64-hundred-million.gmi" rel="nofollow">Marginalia's Index Reaches 100,000,000 Documents</a> &#127882;</dt>
+                <dd>2022-10-21</dd>
+                <dt><a href="https://www.hs.fi/visio/art-2000009139237.html" rel="nofollow">Google ei en&auml;&auml; tide&auml;</a></dt>
                <dd>Helsing Sanomat &#127467;&#127470; 2022-10-19</dd>
-                <dt><a href="https://www.deutschlandfunkkultur.de/google-suche-100.html">Kritik an Googles Suche - Platzhirsch auf dem Nebenschauplatz</a></dt>
+                <dt><a href="https://www.deutschlandfunkkultur.de/google-suche-100.html" rel="nofollow">Kritik an Googles Suche - Platzhirsch auf dem Nebenschauplatz</a></dt>
                <dd>Deutschlandfunk Kultur &#x1f1e9;&#x1f1ea;, 2022-08-18</dd>
                <dt><a href="https://news.ycombinator.com/item?id=31536626" rel="nofollow">Marginalia Goes Open Source</a></dt>
                <dd>Hacker News, 2022-05-28</dd>
                <dt><a href="https://www.youtube.com/watch?v=rTSEr0cRJY8" rel="nofollow">You Should Check Out the Indie Web</a> &#x1F39E;&#xFE0F;</dt>
                <dd>YouTube, You've Got Kat, 2022-03-15 </dd>
-                <dt>
-                    <a href="https://www.newyorker.com/culture/infinite-scroll/what-google-search-isnt-showing-you" rel="nofollow">What Google Search Isn't Showing You</a>
-                </dt>
+                <dt> <a href="https://www.newyorker.com/culture/infinite-scroll/what-google-search-isnt-showing-you" rel="nofollow">What Google Search Isn't Showing You</a> </dt>
                <dd>The New Yorker &#127913;, 2022-03-10</dd>
-                <dt>
-                    <a href="https://www.metafilter.com/194653/Marginalia-Search-Serendipity-Engineering" rel="nofollow">Marginalia Search - Serendipity Engineering</a>
-                </dt>
+                <dt> <a href="https://www.metafilter.com/194653/Marginalia-Search-Serendipity-Engineering" rel="nofollow">Marginalia Search - Serendipity Engineering</a> </dt>
                <dd>MetaFilter, 2022-03-09</dd>
-                <dt>
-                    &#127874; <a href="https://memex.marginalia.nu/log/49-marginalia-1-year.gmi">First anniversary</a>! &#127882;
-                </dt>
-                <dd>
-                    2022-02-26
-                </dd>
-                <dt>
-                    <a href="https://onezero.medium.com/a-search-engine-designed-to-surprise-you-b81944ed5c06" rel="nofollow">A Search Engine Designed To Surprise You</a>
-                </dt>
+                <dt> &#127874; <a href="https://memex.marginalia.nu/log/49-marginalia-1-year.gmi">First anniversary</a>! &#127882; </dt>
+                <dd> 2022-02-26 </dd>
+                <dt> <a href="https://onezero.medium.com/a-search-engine-designed-to-surprise-you-b81944ed5c06" rel="nofollow">A Search Engine Designed To Surprise You</a> </dt>
                <dd>Clive Thompson OneZero, 2021-09-16</dd>
-                <dt>
-                    <a href="https://news.ycombinator.com/item?id=28550764" rel="nofollow">	A search engine that favors text-heavy sites and punishes modern web design</a>
-                </dt>
-                <dd>
-                    Hacker News, 2021-09-16
-                </dd>
+                <dt> <a href="https://news.ycombinator.com/item?id=28550764" rel="nofollow"> A search engine that favors text-heavy sites and punishes modern web design</a> </dt>
+                <dd> Hacker News, 2021-09-16 </dd>
+                <dt>Development begins </dt>
+                <dd>2021-02-26</dd>
            </dl>
        </div>
    </section>
--- a/marginalia_nu/src/main/resources/templates/edge/search-results.hdb
+++ b/marginalia_nu/src/main/resources/templates/edge/search-results.hdb
@ -20,7 +20,9 @@
 {{>edge/parts/search-form}}


+<hr class="w3m-helper" />
 <section class="cards">
+
    {{#if maintenanceMessage}}<section class="card problems onlyscreen"><h2>Maintenance</h2><p class="description">{{maintenanceMessage}}</p></section>{{/if}}
    {{#if evalResult}}<section class="card semantic onlyscreen"><h2>Evaluation</h2><p class="description">{{query}} = {{evalResult}}</p><hr class="w3m-helper" /></section>{{/if}}
    {{#each wiki.entries}}<section class="card semantic onlyscreen"><h2>Encyclopedia</h2><p class="description"><a href="https://encyclopedia.marginalia.nu/wiki/{{.}}"><em>{{.}}</em> Encyclopedia Page</a></p><hr class="w3m-helper" /></section>{{/each}}
--- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocumentTest.java
+++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/loader/SqlLoadProcessedDocumentTest.java
@ -75,7 +75,8 @@ class SqlLoadProcessedDocumentTest {
                EdgeHtmlStandard.HTML5,
                100,
                12345,
-                -3.14
+                -3.14,
+                null
        )));

        var details = dataStoreDao.getUrlDetailsMulti(new EdgeIdArray<>(loaderData.getUrlId(new EdgeUrl("https://www.marginalia.nu/"))));
--- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/processor/logic/PubDateSnifferTest.java
+++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/processor/logic/PubDateSnifferTest.java
@ -0,0 +1,254 @@
+package nu.marginalia.wmsa.edge.converting.processor.logic;
+
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateParser;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.PubDateSniffer;
+import nu.marginalia.wmsa.edge.converting.processor.logic.pubdate.heuristic.PubDateHeuristicDOMParsingPass2;
+import nu.marginalia.wmsa.edge.model.EdgeUrl;
+import nu.marginalia.wmsa.edge.model.crawl.EdgeHtmlStandard;
+import org.jsoup.Jsoup;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class PubDateSnifferTest {
+
+    PubDateSniffer dateSniffer = new PubDateSniffer();
+
+    @Test
+    public void testGetYearFromText() {
+        var ret = PubDateParser.dateFromHighestYearLookingSubstring("&copy; 2005-2010 Bob Dobbs");
+        assertTrue(ret.isPresent());
+        assertEquals(2010, ret.get().year());
+
+        ret = PubDateParser.dateFromHighestYearLookingSubstring("&copy; 99 Bob Dobbs");
+        assertFalse(ret.isPresent());
+
+        ret = PubDateParser.dateFromHighestYearLookingSubstring("&copy; 1939 Bob Dobbs");
+        assertFalse(ret.isPresent());
+
+        ret = PubDateParser.dateFromHighestYearLookingSubstring("In the year 2525, if man is still alive");
+        assertFalse(ret.isPresent());
+    }
+
+    @Test
+    public void testParse() {
+        var ret = PubDateParser.attemptParseDate("2022-01-01");
+        assertTrue(ret.isPresent());
+        assertEquals("2022-01-01", ret.get().dateIso8601());
+        assertEquals(2022, ret.get().year());
+
+        ret = PubDateParser.attemptParseDate("2022-08-24T14:39:14Z");
+        assertTrue(ret.isPresent());
+        assertEquals("2022-08-24", ret.get().dateIso8601());
+        assertEquals(2022, ret.get().year());
+
+        ret = PubDateParser.attemptParseDate("2022-08-24T14:39:14");
+        assertTrue(ret.isPresent());
+        assertEquals("2022-08-24", ret.get().dateIso8601());
+        assertEquals(2022, ret.get().year());
+
+        ret = PubDateParser.attemptParseDate("Sun, 21 Oct 2018 12:16:24 GMT");
+        assertTrue(ret.isPresent());
+        assertEquals("2018-10-21", ret.get().dateIso8601());
+        assertEquals(2018, ret.get().year());
+
+        ret = PubDateParser.attemptParseDate("July 13, 2006");
+        assertTrue(ret.isPresent());
+        assertEquals(2006, ret.get().year());
+
+    }
+
+
+    @Test
+    public void testHtml5A() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <article>
+                        <time pubdate="pubdate" datetime="2022-08-24">time</time>
+                        Wow, sure lor 'em boss
+                        </article>
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals("2022-08-24", ret.dateIso8601());
+    }
+
+    @Test
+    public void testHtml5B() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <time>2022-08-24</time>
+                        Wow, sure lor 'em boss
+                        </article>
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals("2022-08-24", ret.dateIso8601());
+    }
+
+    @Test
+    public void testHtml5C() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <time class="published" datetime="July 13, 2006">July 13, 2006</time>
+                        Wow, sure lor 'em boss
+                        </article>
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals(2006, ret.year());
+    }
+
+    @Test
+    public void testProblemCases() throws IOException, URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse(Files.readString(Path.of("/home/vlofgren/Code/tmp-data/The Switch to Linux Begins .html"))), EdgeHtmlStandard.HTML5, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals(2006, ret.year());
+
+        ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse(Files.readString(Path.of("/home/vlofgren/Code/tmp-data/Black Hat USA 2010 Understanding and Deploying DNSSEC by Paul Wouters and Patrick Nauber.html"))), EdgeHtmlStandard.XHTML, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals(2010, ret.year());
+    }
+
+    @Test
+    public void testGuessYear() {
+        System.out.println(PubDateParser.guessYear(2010, 2020));
+        System.out.println(PubDateParser.guessYear(2010, 2020));
+        System.out.println(PubDateParser.guessYear(2010, 2020));
+        System.out.println(PubDateParser.guessYear(2010, 2020));
+        System.out.println(PubDateParser.guessYear(2010, 2020));
+    }
+
+    @Test
+    public void testMicrodata() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <meta itemprop="datePublished" content="2022-08-24" />
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals("2022-08-24", ret.dateIso8601());
+    }
+
+    @Test
+    public void testRDFa() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <meta property="datePublished" content="2022-08-24" />
+                        """),EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals("2022-08-24", ret.dateIso8601());
+    }
+
+    @Test
+    public void testLD() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <script type="application/ld+json">{"@context":"https:\\/\\/schema.org","@type":"Article","name":"In the Year 2525","url":"https:\\/\\/en.wikipedia.org\\/wiki\\/In_the_Year_2525","sameAs":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","mainEntity":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\\/\\/www.wikimedia.org\\/static\\/images\\/wmf-hor-googpub.png"}},"datePublished":"2004-08-24T14:39:14Z","dateModified":"2022-10-20T11:54:37Z","image":"https:\\/\\/upload.wikimedia.org\\/wikipedia\\/commons\\/4\\/4a\\/In_the_Year_2525_by_Zager_and_Evans_US_vinyl_Side-A_RCA_release.png","headline":"song written and compsoed by Rick Evans, originally recorded by Zager and Evans and released in 1969"}</script><script type="application/ld+json">{"@context":"https:\\/\\/schema.org","@type":"Article","name":"In the Year 2525","url":"https:\\/\\/en.wikipedia.org\\/wiki\\/In_the_Year_2525","sameAs":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","mainEntity":"http:\\/\\/www.wikidata.org\\/entity\\/Q145269","author":{"@type":"Organization","name":"Contributors to Wikimedia projects"},"publisher":{"@type":"Organization","name":"Wikimedia Foundation, Inc.","logo":{"@type":"ImageObject","url":"https:\\/\\/www.wikimedia.org\\/static\\/images\\/wmf-hor-googpub.png"}},"datePublished":"2004-08-24T14:39:14Z","dateModified":"2022-10-20T11:54:37Z","image":"https:\\/\\/upload.wikimedia.org\\/wikipedia\\/commons\\/4\\/4a\\/In_the_Year_2525_by_Zager_and_Evans_US_vinyl_Side-A_RCA_release.png","headline":"song written and compsoed by Rick Evans, originally recorded by Zager and Evans and released in 1969"}</script>
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals("2004-08-24", ret.dateIso8601());
+    }
+
+    @Test
+    public void testPath() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/articles/2022/04/how-to-detect-dates"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <title>No date in the HTML</title>
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertNull(ret.dateIso8601());
+        assertEquals(2022, ret.year());
+    }
+
+    @Test
+    public void testHeader() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("content-type: application/pdf\netag: \"4fc0ba8a7f5090b6fa6be385dca206ec\"\nlast-modified: Thu, 03 Feb 2022 19:22:58 GMT\ncontent-length: 298819\ndate: Wed, 24 Aug 2022 19:48:52 GMT\ncache-control: public, no-transform, immutable, max-age\u003d31536000\naccess-control-expose-headers: Content-Length,Content-Disposition,Content-Range,Etag,Server-Timing,Vary,X-Cld-Error,X-Content-Type-Options\naccess-control-allow-origin: *\naccept-ranges: bytes\ntiming-allow-origin: *\nserver: Cloudinary\nstrict-transport-security: max-age\u003d604800\nx-content-type-options: nosniff\nserver-timing: akam;dur\u003d25;start\u003d2022-08-24T19:48:52.519Z;desc\u003dmiss,rtt;dur\u003d19,cloudinary;dur\u003d129;start\u003d2022-08-23T06:35:17.331Z\n",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <title>No date in the HTML</title>
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertEquals("2022-02-03", ret.dateIso8601());
+    }
+
+
+    @Test
+    public void testDOM() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <p>Published 2003, updated 2022</p>
+                        """), EdgeHtmlStandard.HTML5, true);
+
+        assertFalse(ret.isEmpty());
+        assertNull(ret.dateIso8601());
+        assertEquals(2015, ret.year());
+    }
+
+    @Test
+    public void testCandidate() {
+        System.out.println(PubDateHeuristicDOMParsingPass2.isPossibleCandidate("(C) 2007"));
+        System.out.println(PubDateHeuristicDOMParsingPass2.isPossibleCandidate("(C) 2007-01-01"));
+        System.out.println(PubDateHeuristicDOMParsingPass2.isPossibleCandidate("(C) 01-01.2007"));
+        System.out.println(PubDateHeuristicDOMParsingPass2.isPossibleCandidate("Only $1999"));
+        System.out.println(PubDateHeuristicDOMParsingPass2.isPossibleCandidate("1998B"));
+        System.out.println(PubDateHeuristicDOMParsingPass2.isPossibleCandidate("1998B"));
+        System.out.println(PubDateHeuristicDOMParsingPass2.isPossibleCandidate("2010 black hat ™"));
+    }
+
+    @Test
+    public void testOldInvision() throws URISyntaxException {
+        var ret = dateSniffer.getPubDate("",
+                new EdgeUrl("https://www.example.com/"),
+                Jsoup.parse("""
+                        <!doctype html>
+                        <html>
+                        <div style="float: left;">&nbsp;<b>Post subject:</b> Keyboards.</div><div style="float: right;"><span class="postdetails"><b><img src="./styles/subsilver2/imageset/icon_post_target.gif" width="12" height="9" alt="Post" title="Post" /> <a  href="./viewtopic.php?p=34580&amp;sid=cf0c13dedebb4fea1f03fa73e510cd9f#p34580">#1</a></b></span>&nbsp;<b>Posted:</b> Sun Oct 03, 2010 5:37 pm&nbsp;</div>
+                        """), EdgeHtmlStandard.UNKNOWN, true);
+
+        assertFalse(ret.isEmpty());
+        assertNull(ret.dateIso8601());
+        assertEquals(2010, ret.year());
+    }
+}
--- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/processor/logic/SummaryExtractorTest.java
+++ b/marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/converting/processor/logic/SummaryExtractorTest.java
@ -2,22 +2,91 @@ package nu.marginalia.wmsa.edge.converting.processor.logic;

 import org.jsoup.Jsoup;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;

+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;

 class SummaryExtractorTest {
+    SummaryExtractor summaryExtractor;
+    @BeforeEach
+    public void setUp() {
+        summaryExtractor = new SummaryExtractor(255);
+    }
+
+    @Test
+    public void testSummaryFilter() throws IOException {
+        String html = readClassPathFile("html/monadnock.html");
+        var doc = Jsoup.parse(html);
+        var filter = new SummaryExtractionFilter();
+        doc.filter(filter);
+
+        filter.statistics.entrySet().stream().sorted(Comparator.comparing(e -> -e.getValue().textLength()))
+                .filter(e -> e.getValue().textToTagRatio() > 0.75)
+                .filter(e -> e.getValue().isElement())
+                .filter(e -> e.getValue().textLength() > 32)
+                .filter(e -> e.getValue().pos() < filter.cnt / 2.)
+                .limit(5)
+                .forEach(e -> {
+                    System.out.println(e.getKey().nodeName() + ":" + e.getValue() + " / " + e.getValue().textToTagRatio());
+                    System.out.println(e.getValue().text());
+                });
+    }
+    @Test
+    public void testSummaryFilter3() throws IOException {
+        var data = Path.of("/home/vlofgren/Code/tmp-data/url-327999153");
+        String html = Files.readString(data);
+        var doc = Jsoup.parse(html);
+        var filter = new SummaryExtractionFilter();
+        doc.filter(filter);
+
+        filter.getSummary(255);
+    }
+    @Test
+    public void testSummaryFilter2() throws IOException {
+        var data = Path.of("/home/vlofgren/Code/tmp-data/");
+
+        System.out.println("Running");
+
+        var fos = new PrintWriter(new FileOutputStream("/tmp/summaryDiff.html"));
+        fos.println("<table>");
+
+        for (var file : Objects.requireNonNull(data.toFile().listFiles())) {
+
+            var doc = Jsoup.parse(Files.readString(file.toPath()));
+            fos.println("<tr><th colspan=2>" + file.getName() + "</th></tr>");
+            fos.println("<tr><td width=50%>");
+            var filter = new SummaryExtractionFilter();
+
+            doc.select("header,nav,#header,#nav,#navigation,.header,.nav,.navigation,ul,li").remove();
+            doc.filter(filter);
+            var ret = filter.getSummary(255);
+
+            fos.println(ret);
+            fos.println("</td><td width=50%>");
+            String summary = summaryExtractor.extractSummary(Jsoup.parse(Files.readString(file.toPath())));
+            fos.println(summary);
+            fos.println("</td></tr>");
+        }
+
+        fos.println("</table>");
+        fos.flush();
+    }

    @Test
    void extractSurrey() throws IOException {
        String html = readClassPathFile("html/summarization/surrey.html");
-        SummaryExtractor se = new SummaryExtractor(255);
+        var doc = Jsoup.parse(html);
+        String summary = summaryExtractor.extractSummary(doc);

-        String summary = se.extractSummary(Jsoup.parse(html));

        Assertions.assertFalse(summary.isBlank());

@ -27,9 +96,8 @@ class SummaryExtractorTest {
    @Test
    void extractSurrey1() throws IOException {
        String html = readClassPathFile("html/summarization/surrey.html.1");
-        SummaryExtractor se = new SummaryExtractor(255);
-
-        String summary = se.extractSummary(Jsoup.parse(html));
+        var doc = Jsoup.parse(html);
+        String summary = summaryExtractor.extractSummary(doc);

        Assertions.assertFalse(summary.isBlank());

@ -39,9 +107,8 @@ class SummaryExtractorTest {
    @Test
    void extract187() throws IOException {
        String html = readClassPathFile("html/summarization/187.shtml");
-        SummaryExtractor se = new SummaryExtractor(255);
-
-        String summary = se.extractSummary(Jsoup.parse(html));
+        var doc = Jsoup.parse(html);
+        String summary = summaryExtractor.extractSummary(doc);

        Assertions.assertFalse(summary.isBlank());

@ -51,9 +118,9 @@ class SummaryExtractorTest {
    @Test
    void extractMonadnock() throws IOException {
        String html = readClassPathFile("html/monadnock.html");
-        SummaryExtractor se = new SummaryExtractor(255);

-        String summary = se.extractSummary(Jsoup.parse(html));
+        var doc = Jsoup.parse(html);
+        String summary = summaryExtractor.extractSummary(doc);

        Assertions.assertFalse(summary.isBlank());

@ -63,9 +130,9 @@ class SummaryExtractorTest {
    @Test
    public void testWorkSet() throws IOException {
        var workSet = readWorkSet();
-        SummaryExtractor se = new SummaryExtractor(255);
        workSet.forEach((path, str) -> {
-            String summary = se.extractSummary(Jsoup.parse(str));
+            var doc = Jsoup.parse(str);
+            String summary = summaryExtractor.extractSummary(doc);
            System.out.println(path + ": " + summary);
        });
    }
@ -85,4 +152,5 @@ class SummaryExtractorTest {
        }
        return result;
    }
+
 }