Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(converter) Experimental support for searching by URL

Browse Source 
Add up to synthetic 128 keywords per document, corresponding to links to other websites.
This commit is contained in:
Viktor Lofgren 2024-05-23 17:10:57 +02:00
parent 89aae93e60
commit f83f777fff
2 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
code/processes/converting-process/java/nu/marginalia/converting/processor/logic/links/LinkProcessor.java
View File

@ -32,6 +32,10 @@ public class LinkProcessor {
ret.feedLinks = new ArrayList<>(); ret.feedLinks = new ArrayList<>();
} }
public Set<EdgeUrl> getSeenUrls() {
return seenUrls;
}
public Set<EdgeDomain> getForeignDomains() { public Set<EdgeDomain> getForeignDomains() {
return foreignDomains; return foreignDomains;
} }

12
code/processes/converting-process/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java
View File

@ -295,10 +295,10 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
words.addAllSyntheticTerms(FileLinks.createFileLinkKeywords(lp, domain)); words.addAllSyntheticTerms(FileLinks.createFileLinkKeywords(lp, domain));
words.addAllSyntheticTerms(FileLinks.createFileEndingKeywords(doc)); words.addAllSyntheticTerms(FileLinks.createFileEndingKeywords(doc));
words.addAllSyntheticTerms(createLinkKeywords(lp)); words.addAllSyntheticTerms(createLinkKeywords(lp, domain));
} }
private Set<String> createLinkKeywords(LinkProcessor lp) { private Set<String> createLinkKeywords(LinkProcessor lp, EdgeDomain domain) {
final Set<String> linkTerms = new HashSet<>(); final Set<String> linkTerms = new HashSet<>();
for (var fd : lp.getForeignDomains()) { for (var fd : lp.getForeignDomains()) {
@ -306,6 +306,14 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
linkTerms.add("links:"+fd.getTopDomain().toLowerCase()); linkTerms.add("links:"+fd.getTopDomain().toLowerCase());
} }
// Add keyword terms for the first 128 external links, with no prefix
for (EdgeUrl link : lp.getSeenUrls()) {
if (linkTerms.size() > 128) break;
if (domain.hasSameTopDomain(link.domain)) continue;
linkTerms.add(link.toString());
}
return linkTerms; return linkTerms;
} }