diff --git a/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java b/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java index 3557bed2..d21334b1 100644 --- a/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java +++ b/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java @@ -6,10 +6,13 @@ import com.google.inject.name.Named; import nu.marginalia.index.forward.ForwardIndexConverter; import nu.marginalia.index.forward.ForwardIndexReader; import nu.marginalia.index.journal.reader.IndexJournalReaderSingleCompressedFile; +import nu.marginalia.index.journal.writer.IndexJournalWriter; +import nu.marginalia.index.journal.writer.IndexJournalWriterImpl; import nu.marginalia.index.reverse.ReverseIndexConverter; import nu.marginalia.index.reverse.ReverseIndexPrioReader; import nu.marginalia.index.reverse.ReverseIndexPriorityParameters; import nu.marginalia.index.reverse.ReverseIndexReader; +import nu.marginalia.lexicon.KeywordLexicon; import nu.marginalia.ranking.DomainRankings; import nu.marginalia.index.index.SearchIndexReader; import org.slf4j.Logger; @@ -20,7 +23,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.util.List; import java.util.concurrent.Callable; import java.util.stream.Stream; @@ -92,6 +94,10 @@ public class IndexServicesFactory { ).noneMatch(Files::exists); } + public IndexJournalWriter createIndexJournalWriter(KeywordLexicon lexicon) throws IOException { + return new IndexJournalWriterImpl(lexicon, writerIndexFile.get(LIVE_PART).toPath()); + } + public void convertIndex(DomainRankings domainRankings) throws IOException { convertForwardIndex(domainRankings); convertFullReverseIndex(domainRankings); diff --git a/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java b/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java index 6a46554c..518071a6 100644 --- a/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java +++ b/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java @@ -102,6 +102,11 @@ public class IndexQueryService { } } + // exists for test access + EdgeSearchResultSet justQuery(EdgeSearchSpecification specsSet) { + return new EdgeSearchResultSet(executeSearch(new SearchParameters(specsSet, getSearchSet(specsSet)))); + } + private SearchSet getSearchSet(EdgeSearchSpecification specsSet) { if (specsSet.domains != null && !specsSet.domains.isEmpty()) { return new SmallSearchSet(specsSet.domains); diff --git a/services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTest.java b/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java similarity index 67% rename from services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTest.java rename to services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java index 01af6363..c4956b8b 100644 --- a/services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTest.java +++ b/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java @@ -1,35 +1,65 @@ -package nu.marginalia.index.service; +package nu.marginalia.index.svc; +import com.google.inject.Guice; +import com.google.inject.Inject; +import nu.marginalia.index.client.model.query.EdgeSearchSpecification; +import nu.marginalia.index.client.model.query.EdgeSearchSubquery; +import nu.marginalia.index.client.model.query.SearchSetIdentifier; +import nu.marginalia.index.client.model.results.EdgeSearchResultItem; +import nu.marginalia.index.index.SearchIndex; +import nu.marginalia.index.journal.model.IndexJournalEntryData; +import nu.marginalia.index.journal.model.IndexJournalEntryHeader; +import nu.marginalia.index.journal.writer.IndexJournalWriter; +import nu.marginalia.index.query.limit.QueryLimits; +import nu.marginalia.index.query.limit.QueryStrategy; +import nu.marginalia.index.query.limit.SpecificationLimit; +import nu.marginalia.lexicon.KeywordLexicon; +import nu.marginalia.model.crawl.EdgePageWordFlags; +import nu.marginalia.model.idx.EdgePageDocumentsMetadata; +import nu.marginalia.model.idx.EdgePageWordMetadata; +import nu.marginalia.service.server.Initialization; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; +import spark.Spark; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.stream.IntStream; import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; @Execution(SAME_THREAD) -public class EdgeIndexIntegrationTest { -/* FIXME +public class IndexQueryServiceIntegrationTest { @Inject Initialization initialization; - @Inject - EdgeIndexLexiconService lexiconService; - @Inject - EdgeIndexQueryService queryService; - @Inject - EdgeIndexOpsService opsService; + + IndexQueryServiceIntegrationTestModule testModule; @Inject - SearchIndexControl searchIndexControl; + IndexQueryService queryService; + @Inject + SearchIndex searchIndex; - EdgeIndexIntegrationTestModule testModule; + @Inject + KeywordLexicon keywordLexicon; + + @Inject + IndexJournalWriter indexJournalWriter; @BeforeEach - public void setUp() throws IOException, InterruptedException { + public void setUp() throws IOException { - testModule = new EdgeIndexIntegrationTestModule(); + testModule = new IndexQueryServiceIntegrationTestModule(); Guice.createInjector(testModule).injectMembers(this); initialization.setReady(); - searchIndexControl.initialize(initialization); } @AfterEach @@ -44,12 +74,11 @@ public class EdgeIndexIntegrationTest { for (int i = 1; i < 512; i++) { loadData(i); } - searchIndexControl.getIndexWriter(0).flushWords(); - Thread.sleep(100); - opsService.reindexEndpoint(null, null); + indexJournalWriter.close(); + searchIndex.switchIndex(); - var rsp = queryService.query( + var rsp = queryService.justQuery( EdgeSearchSpecification.builder() .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) .queryStrategy(QueryStrategy.SENTENCE) @@ -77,12 +106,11 @@ public class EdgeIndexIntegrationTest { for (int i = 1; i < 512; i++) { loadDataWithDomain(i/100, i); } - searchIndexControl.getIndexWriter(0).flushWords(); - Thread.sleep(100); - opsService.reindexEndpoint(null, null); + indexJournalWriter.close(); + searchIndex.switchIndex(); - var rsp = queryService.query( + var rsp = queryService.justQuery( EdgeSearchSpecification.builder() .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) .year(SpecificationLimit.none()) @@ -104,12 +132,10 @@ public class EdgeIndexIntegrationTest { for (int i = 1; i < 512; i++) { loadData(i); } - searchIndexControl.getIndexWriter(0).flushWords(); - Thread.sleep(100); + indexJournalWriter.close(); + searchIndex.switchIndex(); - opsService.reindexEndpoint(null, null); - - var rsp = queryService.query( + var rsp = queryService.justQuery( EdgeSearchSpecification.builder() .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) .quality(SpecificationLimit.none()) @@ -142,11 +168,11 @@ public class EdgeIndexIntegrationTest { long[] data = new long[factors.length*2]; for (int i = 0; i < factors.length; i++) { - data[2*i] = lexiconService.getOrInsertWord(Integer.toString(factors[i])); + data[2*i] = keywordLexicon.getOrInsert(Integer.toString(factors[i])); data[2*i + 1] = new EdgePageWordMetadata(i, i, i, EnumSet.of(EdgePageWordFlags.Title)).encode(); } - lexiconService.putWords(0, header, new IndexJournalEntryData(data)); + indexJournalWriter.put(header, new IndexJournalEntryData(data)); } public void loadDataWithDomain(int domain, int id) { @@ -155,11 +181,10 @@ public class EdgeIndexIntegrationTest { long[] data = new long[factors.length*2]; for (int i = 0; i < factors.length; i++) { - data[2*i] = lexiconService.getOrInsertWord(Integer.toString(factors[i])); + data[2*i] = keywordLexicon.getOrInsert(Integer.toString(factors[i])); data[2*i + 1] = new EdgePageWordMetadata(i % 20, i, i, EnumSet.of(EdgePageWordFlags.Title)).encode(); } - lexiconService.putWords(0, header, new IndexJournalEntryData(data)); + indexJournalWriter.put(header, new IndexJournalEntryData(data)); } -*/ } diff --git a/services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTestModule.java b/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java similarity index 59% rename from services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTestModule.java rename to services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java index bf816da3..555492bd 100644 --- a/services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTestModule.java +++ b/services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java @@ -1,10 +1,15 @@ -package nu.marginalia.index.service; +package nu.marginalia.index.svc; import com.google.inject.AbstractModule; import com.google.inject.name.Names; +import nu.marginalia.WmsaHome; import nu.marginalia.index.IndexServicesFactory; +import nu.marginalia.index.journal.writer.IndexJournalWriter; +import nu.marginalia.index.journal.writer.IndexJournalWriterImpl; +import nu.marginalia.lexicon.KeywordLexicon; +import nu.marginalia.lexicon.KeywordLexiconReadOnlyView; +import nu.marginalia.lexicon.journal.KeywordLexiconJournal; import nu.marginalia.ranking.DomainRankings; -import nu.marginalia.index.svc.IndexSearchSetsService; import nu.marginalia.index.svc.searchset.SearchSetAny; import nu.marginalia.index.util.TestUtil; import nu.marginalia.index.client.model.query.SearchSetIdentifier; @@ -17,15 +22,15 @@ import java.util.Random; import static org.mockito.Mockito.when; -public class EdgeIndexIntegrationTestModule extends AbstractModule { +public class IndexQueryServiceIntegrationTestModule extends AbstractModule { Path workDir; Path slowDir; Path fastDir; Random random = new Random(); - public EdgeIndexIntegrationTestModule() throws IOException { - workDir = Files.createTempDirectory(EdgeIndexIntegrationTest.class.getSimpleName()); + public IndexQueryServiceIntegrationTestModule() throws IOException { + workDir = Files.createTempDirectory(IndexQueryServiceIntegrationTest.class.getSimpleName()); slowDir = workDir.resolve("slow"); fastDir = workDir.resolve("fast"); @@ -42,16 +47,22 @@ public class EdgeIndexIntegrationTestModule extends AbstractModule { System.setProperty("small-ram", "true"); try { - bind(IndexServicesFactory.class).toInstance(new IndexServicesFactory(Path.of("/tmp"), + var servicesFactory = new IndexServicesFactory(Path.of("/tmp"), slowDir, fastDir - )); + ); + bind(IndexServicesFactory.class).toInstance(servicesFactory); IndexSearchSetsService setsServiceMock = Mockito.mock(IndexSearchSetsService.class); when(setsServiceMock.getSearchSetByName(SearchSetIdentifier.NONE)).thenReturn(new SearchSetAny()); when(setsServiceMock.getDomainRankings()).thenReturn(new DomainRankings()); - bind(IndexSearchSetsService.class).toInstance(setsServiceMock); + var keywordLexicon = new KeywordLexicon(new KeywordLexiconJournal(slowDir.resolve("dictionary.dat").toFile())); + bind(KeywordLexicon.class).toInstance(keywordLexicon); + bind(KeywordLexiconReadOnlyView.class).toInstance(new KeywordLexiconReadOnlyView(keywordLexicon)); + + bind(IndexJournalWriter.class).toInstance(servicesFactory.createIndexJournalWriter(keywordLexicon)); + bind(String.class).annotatedWith(Names.named("service-host")).toInstance("127.0.0.1"); bind(Integer.class).annotatedWith(Names.named("service-port")).toProvider(this::randomPort); } catch (IOException e) {