Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move list-conversion into getDescription method.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-06-10 11:03:41 +02:00 committed by Viktor
parent 88399e30e2
commit 44b1fe0e6d
1 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java
View File

@ -135,9 +135,7 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
DocumentKeywordsBuilder words = keywordExtractor.extractKeywords(dld, url); DocumentKeywordsBuilder words = keywordExtractor.extractKeywords(dld, url);
ret.description = getDescription(doc, ret.description = getDescription(doc, words.importantWords);
new ArrayList<>(words.importantWords)
);
var tagWords = new MetaTagsBuilder() var tagWords = new MetaTagsBuilder()
.addDomainCrawlData(crawledDomain) .addDomainCrawlData(crawledDomain)
@ -270,11 +268,20 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
} }
private String getDescription(Document doc, private String getDescription(Document doc,
Collection<String> importantWords) Set<String> importantWords)
{ {
importantWords.removeIf(w -> w.contains("_")); List<String> cleanedWords = new ArrayList<>(importantWords.size());
return summaryExtractor.extractSummary(doc, importantWords); for (var word : importantWords) {
// summary extraction is not interested in n-grams
if (word.contains("_")) {
continue;
}
cleanedWords.add(word);
}
return summaryExtractor.extractSummary(doc, cleanedWords);
} }
private int getLength(Document doc) { private int getLength(Document doc) {