From 54ed3b86bae11bcfe6fb8daa09bff4bd15968dbd Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 15 Dec 2023 21:49:35 +0100 Subject: [PATCH] (minor) Remove dead code. --- .../marginalia/crawl/retreival/CrawlerRetreiver.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java index 35f5bcd0..18035d52 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/retreival/CrawlerRetreiver.java @@ -312,17 +312,6 @@ public class CrawlerRetreiver implements AutoCloseable { || proto.equalsIgnoreCase("https"); } - // FIXME this does not belong in the crawler - private Optional findCanonicalUrl(EdgeUrl baseUrl, Document parsed) { - baseUrl = baseUrl.domain.toRootUrl(); - - for (var link : parsed.select("link[rel=canonical]")) { - return linkParser.parseLink(baseUrl, link); - } - - return Optional.empty(); - } - private String findIp(String domain) { try { return InetAddress.getByName(domain).getHostAddress();