Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move list-conversion into getDescription method.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-06-10 11:03:41 +02:00 committed by Viktor
parent 88399e30e2
commit 44b1fe0e6d
1 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/HtmlDocumentProcessorPlugin.java
View File

@ -135,9 +135,7 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
DocumentKeywordsBuilder words = keywordExtractor.extractKeywords(dld, url);
ret.description = getDescription(doc,
new ArrayList<>(words.importantWords)
);
ret.description = getDescription(doc, words.importantWords);
var tagWords = new MetaTagsBuilder()
.addDomainCrawlData(crawledDomain)
@ -270,11 +268,20 @@ public class HtmlDocumentProcessorPlugin extends AbstractDocumentProcessorPlugin
}
private String getDescription(Document doc,
Collection<String> importantWords)
Set<String> importantWords)
{
importantWords.removeIf(w -> w.contains("_"));
List<String> cleanedWords = new ArrayList<>(importantWords.size());
return summaryExtractor.extractSummary(doc, importantWords);
for (var word : importantWords) {
// summary extraction is not interested in n-grams
if (word.contains("_")) {
continue;
}
cleanedWords.add(word);
}
return summaryExtractor.extractSummary(doc, cleanedWords);
}
private int getLength(Document doc) {