Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(term-frequency) Fix concurrency issues in SentenceExtractor and TermFrequencyExporter

Browse Source 
How'd This Ever Work? (tm)

TermFrequencyExporter was using Math.clamp() incorrectly, and SentenceExtractor was synchronizing on its own instance when initializing shared static members, causing rare issues when spinning multiple SE:s up at once.
This commit is contained in:
Viktor Lofgren 2024-07-15 05:15:30 +02:00
parent 6401a513d7
commit ffd970036d
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/features-convert/data-extractors/java/nu/marginalia/extractor/TermFrequencyExporter.java
View File

@ -54,7 +54,7 @@ public class TermFrequencyExporter implements ExporterIf {
TLongIntHashMap counts = new TLongIntHashMap(100_000_000, 0.7f, -1, -1); TLongIntHashMap counts = new TLongIntHashMap(100_000_000, 0.7f, -1, -1);
AtomicInteger docCount = new AtomicInteger(); AtomicInteger docCount = new AtomicInteger();
SimpleBlockingThreadPool sjp = new SimpleBlockingThreadPool("exporter", Math.clamp(2, 16, Runtime.getRuntime().availableProcessors() / 2), 4); SimpleBlockingThreadPool sjp = new SimpleBlockingThreadPool("exporter", Math.clamp(Runtime.getRuntime().availableProcessors() / 2, 2, 16), 4);
Path crawlerLogFile = inputDir.resolve("crawler.log"); Path crawlerLogFile = inputDir.resolve("crawler.log");
for (var item : WorkLog.iterable(crawlerLogFile)) { for (var item : WorkLog.iterable(crawlerLogFile)) {

2
code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceExtractor.java
View File

@ -59,7 +59,7 @@ public class SentenceExtractor {
logger.error("Could not initialize sentence detector", ex); logger.error("Could not initialize sentence detector", ex);
} }
synchronized (this) { synchronized (SentenceExtractor.class) {
if (ngramLexicon == null) { if (ngramLexicon == null) {
ngramLexicon = new NgramLexicon(models); ngramLexicon = new NgramLexicon(models);
} }