Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(term-freq-exporter) Extract ngrams in term-frequency-exporter

Browse Source
This commit is contained in:
Viktor Lofgren 2024-04-10 16:58:05 +02:00
parent fcdc843c15
commit 4b47fadbab
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
code/features-convert/data-extractors/java/nu/marginalia/extractor/TermFrequencyExporter.java
View File

@ -127,6 +127,10 @@ public class TermFrequencyExporter implements ExporterIf {
for (var word : sent) { for (var word : sent) {
words.add(longHash(word.stemmed().getBytes(StandardCharsets.UTF_8))); words.add(longHash(word.stemmed().getBytes(StandardCharsets.UTF_8)));
} }
for (var ngram : sent.ngramStemmed) {
words.add(longHash(ngram.getBytes()));
}
} }
synchronized (counts) { synchronized (counts) {