Repair integration tests

services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java

@@ -6,10 +6,13 @@ import com.google.inject.name.Named;
 import nu.marginalia.index.forward.ForwardIndexConverter;
 import nu.marginalia.index.forward.ForwardIndexReader;
 import nu.marginalia.index.journal.reader.IndexJournalReaderSingleCompressedFile;
+import nu.marginalia.index.journal.writer.IndexJournalWriter;
+import nu.marginalia.index.journal.writer.IndexJournalWriterImpl;
 import nu.marginalia.index.reverse.ReverseIndexConverter;
 import nu.marginalia.index.reverse.ReverseIndexPrioReader;
 import nu.marginalia.index.reverse.ReverseIndexPriorityParameters;
 import nu.marginalia.index.reverse.ReverseIndexReader;
+import nu.marginalia.lexicon.KeywordLexicon;
 import nu.marginalia.ranking.DomainRankings;
 import nu.marginalia.index.index.SearchIndexReader;
 import org.slf4j.Logger;
@@ -20,7 +23,6 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
-import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.stream.Stream;
 
@@ -92,6 +94,10 @@ public class IndexServicesFactory {
         ).noneMatch(Files::exists);
     }
 
+    public IndexJournalWriter createIndexJournalWriter(KeywordLexicon lexicon) throws IOException {
+        return new IndexJournalWriterImpl(lexicon, writerIndexFile.get(LIVE_PART).toPath());
+    }
+
     public void convertIndex(DomainRankings domainRankings) throws IOException {
         convertForwardIndex(domainRankings);
         convertFullReverseIndex(domainRankings);

services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java

@@ -102,6 +102,11 @@ public class IndexQueryService {
         }
     }
 
+    // exists for test access
+    EdgeSearchResultSet justQuery(EdgeSearchSpecification specsSet) {
+        return new EdgeSearchResultSet(executeSearch(new SearchParameters(specsSet, getSearchSet(specsSet))));
+    }
+
     private SearchSet getSearchSet(EdgeSearchSpecification specsSet) {
         if (specsSet.domains != null && !specsSet.domains.isEmpty()) {
             return new SmallSearchSet(specsSet.domains);

services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java

@@ -1,35 +1,65 @@
-package nu.marginalia.index.service;
+package nu.marginalia.index.svc;
 
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import nu.marginalia.index.client.model.query.EdgeSearchSpecification;
+import nu.marginalia.index.client.model.query.EdgeSearchSubquery;
+import nu.marginalia.index.client.model.query.SearchSetIdentifier;
+import nu.marginalia.index.client.model.results.EdgeSearchResultItem;
+import nu.marginalia.index.index.SearchIndex;
+import nu.marginalia.index.journal.model.IndexJournalEntryData;
+import nu.marginalia.index.journal.model.IndexJournalEntryHeader;
+import nu.marginalia.index.journal.writer.IndexJournalWriter;
+import nu.marginalia.index.query.limit.QueryLimits;
+import nu.marginalia.index.query.limit.QueryStrategy;
+import nu.marginalia.index.query.limit.SpecificationLimit;
+import nu.marginalia.lexicon.KeywordLexicon;
+import nu.marginalia.model.crawl.EdgePageWordFlags;
+import nu.marginalia.model.idx.EdgePageDocumentsMetadata;
+import nu.marginalia.model.idx.EdgePageWordMetadata;
+import nu.marginalia.service.server.Initialization;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.parallel.Execution;
+import spark.Spark;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.stream.IntStream;
 
 import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
 
 @Execution(SAME_THREAD)
-public class EdgeIndexIntegrationTest {
+public class IndexQueryServiceIntegrationTest {
-/* FIXME
 
     @Inject
     Initialization initialization;
-    @Inject
+
-    EdgeIndexLexiconService lexiconService;
+    IndexQueryServiceIntegrationTestModule testModule;
-    @Inject
-    EdgeIndexQueryService queryService;
-    @Inject
-    EdgeIndexOpsService opsService;
 
     @Inject
-    SearchIndexControl searchIndexControl;
+    IndexQueryService queryService;
+    @Inject
+    SearchIndex searchIndex;
 
-    EdgeIndexIntegrationTestModule testModule;
+    @Inject
+    KeywordLexicon keywordLexicon;
+
+    @Inject
+    IndexJournalWriter indexJournalWriter;
 
     @BeforeEach
-    public void setUp() throws IOException, InterruptedException {
+    public void setUp() throws IOException {
 
-        testModule = new EdgeIndexIntegrationTestModule();
+        testModule = new IndexQueryServiceIntegrationTestModule();
         Guice.createInjector(testModule).injectMembers(this);
 
         initialization.setReady();
-        searchIndexControl.initialize(initialization);
     }
 
     @AfterEach
@@ -44,12 +74,11 @@ public class EdgeIndexIntegrationTest {
         for (int i = 1; i < 512; i++) {
             loadData(i);
         }
-        searchIndexControl.getIndexWriter(0).flushWords();
-        Thread.sleep(100);
 
-        opsService.reindexEndpoint(null, null);
+        indexJournalWriter.close();
+        searchIndex.switchIndex();
 
-        var rsp = queryService.query(
+        var rsp = queryService.justQuery(
                 EdgeSearchSpecification.builder()
                         .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
                         .queryStrategy(QueryStrategy.SENTENCE)
@@ -77,12 +106,11 @@ public class EdgeIndexIntegrationTest {
         for (int i = 1; i < 512; i++) {
             loadDataWithDomain(i/100, i);
         }
-        searchIndexControl.getIndexWriter(0).flushWords();
-        Thread.sleep(100);
 
-        opsService.reindexEndpoint(null, null);
+        indexJournalWriter.close();
+        searchIndex.switchIndex();
 
-        var rsp = queryService.query(
+        var rsp = queryService.justQuery(
                 EdgeSearchSpecification.builder()
                         .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
                         .year(SpecificationLimit.none())
@@ -104,12 +132,10 @@ public class EdgeIndexIntegrationTest {
         for (int i = 1; i < 512; i++) {
             loadData(i);
         }
-        searchIndexControl.getIndexWriter(0).flushWords();
+        indexJournalWriter.close();
-        Thread.sleep(100);
+        searchIndex.switchIndex();
 
-        opsService.reindexEndpoint(null, null);
+        var rsp = queryService.justQuery(
-
-        var rsp = queryService.query(
                 EdgeSearchSpecification.builder()
                         .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
                         .quality(SpecificationLimit.none())
@@ -142,11 +168,11 @@ public class EdgeIndexIntegrationTest {
 
         long[] data = new long[factors.length*2];
         for (int i = 0; i < factors.length; i++) {
-            data[2*i] = lexiconService.getOrInsertWord(Integer.toString(factors[i]));
+            data[2*i] = keywordLexicon.getOrInsert(Integer.toString(factors[i]));
             data[2*i + 1] = new EdgePageWordMetadata(i, i, i, EnumSet.of(EdgePageWordFlags.Title)).encode();
         }
 
-        lexiconService.putWords(0, header, new IndexJournalEntryData(data));
+        indexJournalWriter.put(header, new IndexJournalEntryData(data));
     }
 
     public void loadDataWithDomain(int domain, int id) {
@@ -155,11 +181,10 @@ public class EdgeIndexIntegrationTest {
 
         long[] data = new long[factors.length*2];
         for (int i = 0; i < factors.length; i++) {
-            data[2*i] = lexiconService.getOrInsertWord(Integer.toString(factors[i]));
+            data[2*i] = keywordLexicon.getOrInsert(Integer.toString(factors[i]));
             data[2*i + 1] = new EdgePageWordMetadata(i % 20, i, i, EnumSet.of(EdgePageWordFlags.Title)).encode();
         }
 
-        lexiconService.putWords(0, header, new IndexJournalEntryData(data));
+        indexJournalWriter.put(header, new IndexJournalEntryData(data));
     }
-*/
 }

services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java

@@ -1,10 +1,15 @@
-package nu.marginalia.index.service;
+package nu.marginalia.index.svc;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.name.Names;
+import nu.marginalia.WmsaHome;
 import nu.marginalia.index.IndexServicesFactory;
+import nu.marginalia.index.journal.writer.IndexJournalWriter;
+import nu.marginalia.index.journal.writer.IndexJournalWriterImpl;
+import nu.marginalia.lexicon.KeywordLexicon;
+import nu.marginalia.lexicon.KeywordLexiconReadOnlyView;
+import nu.marginalia.lexicon.journal.KeywordLexiconJournal;
 import nu.marginalia.ranking.DomainRankings;
-import nu.marginalia.index.svc.IndexSearchSetsService;
 import nu.marginalia.index.svc.searchset.SearchSetAny;
 import nu.marginalia.index.util.TestUtil;
 import nu.marginalia.index.client.model.query.SearchSetIdentifier;
@@ -17,15 +22,15 @@ import java.util.Random;
 
 import static org.mockito.Mockito.when;
 
-public class EdgeIndexIntegrationTestModule extends AbstractModule {
+public class IndexQueryServiceIntegrationTestModule extends AbstractModule {
     Path workDir;
     Path slowDir;
     Path fastDir;
 
     Random random = new Random();
 
-    public EdgeIndexIntegrationTestModule() throws IOException {
+    public IndexQueryServiceIntegrationTestModule() throws IOException {
-        workDir = Files.createTempDirectory(EdgeIndexIntegrationTest.class.getSimpleName());
+        workDir = Files.createTempDirectory(IndexQueryServiceIntegrationTest.class.getSimpleName());
         slowDir = workDir.resolve("slow");
         fastDir = workDir.resolve("fast");
 
@@ -42,16 +47,22 @@ public class EdgeIndexIntegrationTestModule extends AbstractModule {
 
         System.setProperty("small-ram", "true");
         try {
-            bind(IndexServicesFactory.class).toInstance(new IndexServicesFactory(Path.of("/tmp"),
+            var servicesFactory = new IndexServicesFactory(Path.of("/tmp"),
                     slowDir, fastDir
-            ));
+            );
+            bind(IndexServicesFactory.class).toInstance(servicesFactory);
 
             IndexSearchSetsService setsServiceMock = Mockito.mock(IndexSearchSetsService.class);
             when(setsServiceMock.getSearchSetByName(SearchSetIdentifier.NONE)).thenReturn(new SearchSetAny());
             when(setsServiceMock.getDomainRankings()).thenReturn(new DomainRankings());
-
             bind(IndexSearchSetsService.class).toInstance(setsServiceMock);
 
+            var keywordLexicon = new KeywordLexicon(new KeywordLexiconJournal(slowDir.resolve("dictionary.dat").toFile()));
+            bind(KeywordLexicon.class).toInstance(keywordLexicon);
+            bind(KeywordLexiconReadOnlyView.class).toInstance(new KeywordLexiconReadOnlyView(keywordLexicon));
+
+            bind(IndexJournalWriter.class).toInstance(servicesFactory.createIndexJournalWriter(keywordLexicon));
+
             bind(String.class).annotatedWith(Names.named("service-host")).toInstance("127.0.0.1");
             bind(Integer.class).annotatedWith(Names.named("service-port")).toProvider(this::randomPort);
         } catch (IOException e) {