Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Repair integration tests

Browse Source
This commit is contained in:
Viktor Lofgren 2023-03-05 12:24:12 +01:00
parent cf00963e57
commit 96f6cd19e9
4 changed files with 88 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java
View File

@ -6,10 +6,13 @@ import com.google.inject.name.Named;
import nu.marginalia.index.forward.ForwardIndexConverter;
import nu.marginalia.index.forward.ForwardIndexReader;
import nu.marginalia.index.journal.reader.IndexJournalReaderSingleCompressedFile;
import nu.marginalia.index.journal.writer.IndexJournalWriter;
import nu.marginalia.index.journal.writer.IndexJournalWriterImpl;
import nu.marginalia.index.reverse.ReverseIndexConverter;
import nu.marginalia.index.reverse.ReverseIndexPrioReader;
import nu.marginalia.index.reverse.ReverseIndexPriorityParameters;
import nu.marginalia.index.reverse.ReverseIndexReader;
import nu.marginalia.lexicon.KeywordLexicon;
import nu.marginalia.ranking.DomainRankings;
import nu.marginalia.index.index.SearchIndexReader;
import org.slf4j.Logger;
@ -20,7 +23,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.stream.Stream;
@ -92,6 +94,10 @@ public class IndexServicesFactory {
).noneMatch(Files::exists);
}
public IndexJournalWriter createIndexJournalWriter(KeywordLexicon lexicon) throws IOException {
return new IndexJournalWriterImpl(lexicon, writerIndexFile.get(LIVE_PART).toPath());
}
public void convertIndex(DomainRankings domainRankings) throws IOException {
convertForwardIndex(domainRankings);
convertFullReverseIndex(domainRankings);

5
services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java
View File

@ -102,6 +102,11 @@ public class IndexQueryService {
}
}
// exists for test access
EdgeSearchResultSet justQuery(EdgeSearchSpecification specsSet) {
return new EdgeSearchResultSet(executeSearch(new SearchParameters(specsSet, getSearchSet(specsSet))));
}
private SearchSet getSearchSet(EdgeSearchSpecification specsSet) {
if (specsSet.domains != null && !specsSet.domains.isEmpty()) {
return new SmallSearchSet(specsSet.domains);

89
services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTest.java → services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTest.java
View File

@ -1,35 +1,65 @@
package nu.marginalia.index.service;
package nu.marginalia.index.svc;
import com.google.inject.Guice;
import com.google.inject.Inject;
import nu.marginalia.index.client.model.query.EdgeSearchSpecification;
import nu.marginalia.index.client.model.query.EdgeSearchSubquery;
import nu.marginalia.index.client.model.query.SearchSetIdentifier;
import nu.marginalia.index.client.model.results.EdgeSearchResultItem;
import nu.marginalia.index.index.SearchIndex;
import nu.marginalia.index.journal.model.IndexJournalEntryData;
import nu.marginalia.index.journal.model.IndexJournalEntryHeader;
import nu.marginalia.index.journal.writer.IndexJournalWriter;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.lexicon.KeywordLexicon;
import nu.marginalia.model.crawl.EdgePageWordFlags;
import nu.marginalia.model.idx.EdgePageDocumentsMetadata;
import nu.marginalia.model.idx.EdgePageWordMetadata;
import nu.marginalia.service.server.Initialization;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import spark.Spark;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.stream.IntStream;
import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
@Execution(SAME_THREAD)
public class EdgeIndexIntegrationTest {
/* FIXME
public class IndexQueryServiceIntegrationTest {
@Inject
Initialization initialization;
@Inject
EdgeIndexLexiconService lexiconService;
@Inject
EdgeIndexQueryService queryService;
@Inject
EdgeIndexOpsService opsService;
IndexQueryServiceIntegrationTestModule testModule;
@Inject
SearchIndexControl searchIndexControl;
IndexQueryService queryService;
@Inject
SearchIndex searchIndex;
EdgeIndexIntegrationTestModule testModule;
@Inject
KeywordLexicon keywordLexicon;
@Inject
IndexJournalWriter indexJournalWriter;
@BeforeEach
public void setUp() throws IOException, InterruptedException {
public void setUp() throws IOException {
testModule = new EdgeIndexIntegrationTestModule();
testModule = new IndexQueryServiceIntegrationTestModule();
Guice.createInjector(testModule).injectMembers(this);
initialization.setReady();
searchIndexControl.initialize(initialization);
}
@AfterEach
@ -44,12 +74,11 @@ public class EdgeIndexIntegrationTest {
for (int i = 1; i < 512; i++) {
loadData(i);
}
searchIndexControl.getIndexWriter(0).flushWords();
Thread.sleep(100);
opsService.reindexEndpoint(null, null);
indexJournalWriter.close();
searchIndex.switchIndex();
var rsp = queryService.query(
var rsp = queryService.justQuery(
EdgeSearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.queryStrategy(QueryStrategy.SENTENCE)
@ -77,12 +106,11 @@ public class EdgeIndexIntegrationTest {
for (int i = 1; i < 512; i++) {
loadDataWithDomain(i/100, i);
}
searchIndexControl.getIndexWriter(0).flushWords();
Thread.sleep(100);
opsService.reindexEndpoint(null, null);
indexJournalWriter.close();
searchIndex.switchIndex();
var rsp = queryService.query(
var rsp = queryService.justQuery(
EdgeSearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.year(SpecificationLimit.none())
@ -104,12 +132,10 @@ public class EdgeIndexIntegrationTest {
for (int i = 1; i < 512; i++) {
loadData(i);
}
searchIndexControl.getIndexWriter(0).flushWords();
Thread.sleep(100);
indexJournalWriter.close();
searchIndex.switchIndex();
opsService.reindexEndpoint(null, null);
var rsp = queryService.query(
var rsp = queryService.justQuery(
EdgeSearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.quality(SpecificationLimit.none())
@ -142,11 +168,11 @@ public class EdgeIndexIntegrationTest {
long[] data = new long[factors.length*2];
for (int i = 0; i < factors.length; i++) {
data[2*i] = lexiconService.getOrInsertWord(Integer.toString(factors[i]));
data[2*i] = keywordLexicon.getOrInsert(Integer.toString(factors[i]));
data[2*i + 1] = new EdgePageWordMetadata(i, i, i, EnumSet.of(EdgePageWordFlags.Title)).encode();
}
lexiconService.putWords(0, header, new IndexJournalEntryData(data));
indexJournalWriter.put(header, new IndexJournalEntryData(data));
}
public void loadDataWithDomain(int domain, int id) {
@ -155,11 +181,10 @@ public class EdgeIndexIntegrationTest {
long[] data = new long[factors.length*2];
for (int i = 0; i < factors.length; i++) {
data[2*i] = lexiconService.getOrInsertWord(Integer.toString(factors[i]));
data[2*i] = keywordLexicon.getOrInsert(Integer.toString(factors[i]));
data[2*i + 1] = new EdgePageWordMetadata(i % 20, i, i, EnumSet.of(EdgePageWordFlags.Title)).encode();
}
lexiconService.putWords(0, header, new IndexJournalEntryData(data));
indexJournalWriter.put(header, new IndexJournalEntryData(data));
}
*/
}

27
services-core/index-service/src/test/java/nu/marginalia/index/service/EdgeIndexIntegrationTestModule.java → services-core/index-service/src/test/java/nu/marginalia/index/svc/IndexQueryServiceIntegrationTestModule.java
View File

@ -1,10 +1,15 @@
package nu.marginalia.index.service;
package nu.marginalia.index.svc;
import com.google.inject.AbstractModule;
import com.google.inject.name.Names;
import nu.marginalia.WmsaHome;
import nu.marginalia.index.IndexServicesFactory;
import nu.marginalia.index.journal.writer.IndexJournalWriter;
import nu.marginalia.index.journal.writer.IndexJournalWriterImpl;
import nu.marginalia.lexicon.KeywordLexicon;
import nu.marginalia.lexicon.KeywordLexiconReadOnlyView;
import nu.marginalia.lexicon.journal.KeywordLexiconJournal;
import nu.marginalia.ranking.DomainRankings;
import nu.marginalia.index.svc.IndexSearchSetsService;
import nu.marginalia.index.svc.searchset.SearchSetAny;
import nu.marginalia.index.util.TestUtil;
import nu.marginalia.index.client.model.query.SearchSetIdentifier;
@ -17,15 +22,15 @@ import java.util.Random;
import static org.mockito.Mockito.when;
public class EdgeIndexIntegrationTestModule extends AbstractModule {
public class IndexQueryServiceIntegrationTestModule extends AbstractModule {
Path workDir;
Path slowDir;
Path fastDir;
Random random = new Random();
public EdgeIndexIntegrationTestModule() throws IOException {
workDir = Files.createTempDirectory(EdgeIndexIntegrationTest.class.getSimpleName());
public IndexQueryServiceIntegrationTestModule() throws IOException {
workDir = Files.createTempDirectory(IndexQueryServiceIntegrationTest.class.getSimpleName());
slowDir = workDir.resolve("slow");
fastDir = workDir.resolve("fast");
@ -42,16 +47,22 @@ public class EdgeIndexIntegrationTestModule extends AbstractModule {
System.setProperty("small-ram", "true");
try {
bind(IndexServicesFactory.class).toInstance(new IndexServicesFactory(Path.of("/tmp"),
var servicesFactory = new IndexServicesFactory(Path.of("/tmp"),
slowDir, fastDir
));
);
bind(IndexServicesFactory.class).toInstance(servicesFactory);
IndexSearchSetsService setsServiceMock = Mockito.mock(IndexSearchSetsService.class);
when(setsServiceMock.getSearchSetByName(SearchSetIdentifier.NONE)).thenReturn(new SearchSetAny());
when(setsServiceMock.getDomainRankings()).thenReturn(new DomainRankings());
bind(IndexSearchSetsService.class).toInstance(setsServiceMock);
var keywordLexicon = new KeywordLexicon(new KeywordLexiconJournal(slowDir.resolve("dictionary.dat").toFile()));
bind(KeywordLexicon.class).toInstance(keywordLexicon);
bind(KeywordLexiconReadOnlyView.class).toInstance(new KeywordLexiconReadOnlyView(keywordLexicon));
bind(IndexJournalWriter.class).toInstance(servicesFactory.createIndexJournalWriter(keywordLexicon));
bind(String.class).annotatedWith(Names.named("service-host")).toInstance("127.0.0.1");
bind(Integer.class).annotatedWith(Names.named("service-port")).toProvider(this::randomPort);
} catch (IOException e) {