Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(term-freq-exporter) Extract ngrams in term-frequency-exporter

Browse Source
This commit is contained in:
Viktor Lofgren 2024-04-10 16:58:05 +02:00
parent 4fb86ac692
commit 491d6bec46
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
code/features-convert/data-extractors/java/nu/marginalia/extractor/TermFrequencyExporter.java
View File

@ -127,6 +127,10 @@ public class TermFrequencyExporter implements ExporterIf {
for (var word : sent) {
words.add(longHash(word.stemmed().getBytes(StandardCharsets.UTF_8)));
}
for (var ngram : sent.ngramStemmed) {
words.add(longHash(ngram.getBytes()));
}
}
synchronized (counts) {