From ccc41d171721fee7cf2970f02dd5963a9e651df9 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Mon, 10 Apr 2023 14:50:57 +0200 Subject: [PATCH] Clean up of the index query handling related code. --- .../nu/marginalia/index/query/IndexQuery.java | 7 ++- .../index/query/IndexQueryPriority.java | 14 ++++++ .../full/ReverseIndexFullEntrySource.java | 13 +---- .../index/full/ReverseIndexFullReader.java | 5 +- .../ReverseIndexPriorityEntrySource.java | 12 +---- .../priority/ReverseIndexPriorityReader.java | 3 +- .../ReverseIndexEntrySourceBehavior.java | 11 ----- .../ReverseIndexFullConverterTest.java | 9 ++-- .../ReverseIndexFullConverterTest2.java | 5 +- .../marginalia/index/index/SearchIndex.java | 37 ++++++++------ .../index/index/SearchIndexQueryBuilder.java | 2 +- .../index/index/SearchIndexReader.java | 24 +++++----- .../index/svc/IndexQueryService.java | 48 +++++++++++++------ 13 files changed, 101 insertions(+), 89 deletions(-) create mode 100644 code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java delete mode 100644 code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexEntrySourceBehavior.java diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java index 97d76e4e..aa4102c8 100644 --- a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java +++ b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java @@ -10,9 +10,14 @@ import java.util.stream.Collectors; public class IndexQuery { private final List sources; private final List inclusionFilter = new ArrayList<>(10); + public final IndexQueryPriority queryPriority; - public IndexQuery(List sources) { + public final int fetchSizeMultiplier; + + public IndexQuery(List sources, IndexQueryPriority priority, int fetchSizeMultiplier) { this.sources = sources; + this.queryPriority = priority; + this.fetchSizeMultiplier = fetchSizeMultiplier; } public void addInclusionFilter(QueryFilterStepIf filter) { diff --git a/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java new file mode 100644 index 00000000..b2f37350 --- /dev/null +++ b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryPriority.java @@ -0,0 +1,14 @@ +package nu.marginalia.index.query; + +/** Designates the presumptive value of an IndexQuery. + */ +public enum IndexQueryPriority { + /** This is likely to produce highly relevant results */ + BEST, + + /** This may produce relevant results */ + GOOD, + + /** This is a fallback query, only execute if no higher prioritized query returned any results */ + FALLBACK +} diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullEntrySource.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullEntrySource.java index 186d057c..8e7c6b01 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullEntrySource.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullEntrySource.java @@ -3,7 +3,6 @@ package nu.marginalia.index.full; import nu.marginalia.array.buffer.LongQueryBuffer; import nu.marginalia.btree.BTreeReader; import nu.marginalia.index.query.EntrySource; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; import static java.lang.Math.min; @@ -15,14 +14,11 @@ public class ReverseIndexFullEntrySource implements EntrySource { final int entrySize; private final int wordId; - private final ReverseIndexEntrySourceBehavior behavior; public ReverseIndexFullEntrySource(BTreeReader reader, int entrySize, - ReverseIndexEntrySourceBehavior behavior, int wordId) { this.reader = reader; - this.behavior = behavior; this.entrySize = entrySize; this.wordId = wordId; @@ -37,13 +33,6 @@ public class ReverseIndexFullEntrySource implements EntrySource { @Override public void read(LongQueryBuffer buffer) { - if (behavior == ReverseIndexEntrySourceBehavior.DO_NOT_PREFER - && buffer.hasRetainedData()) - { - pos = endOffset; - return; - } - buffer.end = min(buffer.end, endOffset - pos); reader.readData(buffer.data, buffer.end, pos); pos += buffer.end; @@ -71,6 +60,6 @@ public class ReverseIndexFullEntrySource implements EntrySource { @Override public String indexName() { - return "Priority:" + wordId; + return "Full:" + wordId; } } diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullReader.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullReader.java index e6206aa4..6342c436 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullReader.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/full/ReverseIndexFullReader.java @@ -1,6 +1,5 @@ package nu.marginalia.index.full; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; import nu.marginalia.index.query.ReverseIndexRejectFilter; import nu.marginalia.index.query.ReverseIndexRetainFilter; import nu.marginalia.array.LongArray; @@ -51,7 +50,7 @@ public class ReverseIndexFullReader { return createReaderNew(offset).findEntry(documentId) >= 0; } - public EntrySource documents(int wordId, ReverseIndexEntrySourceBehavior behavior) { + public EntrySource documents(int wordId) { if (null == words) { logger.warn("Reverse index is not ready, dropping query"); return new EmptyEntrySource(); @@ -63,7 +62,7 @@ public class ReverseIndexFullReader { if (offset < 0) return new EmptyEntrySource(); - return new ReverseIndexFullEntrySource(createReaderNew(offset), ReverseIndexFullParameters.ENTRY_SIZE, behavior, wordId); + return new ReverseIndexFullEntrySource(createReaderNew(offset), ReverseIndexFullParameters.ENTRY_SIZE, wordId); } public QueryFilterStepIf also(int wordId) { diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityEntrySource.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityEntrySource.java index e971ca83..f4cc932d 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityEntrySource.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityEntrySource.java @@ -3,7 +3,6 @@ package nu.marginalia.index.priority; import nu.marginalia.array.buffer.LongQueryBuffer; import nu.marginalia.btree.BTreeReader; import nu.marginalia.index.query.EntrySource; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; import static java.lang.Math.min; @@ -13,12 +12,10 @@ public class ReverseIndexPriorityEntrySource implements EntrySource { int pos; int endOffset; - private final ReverseIndexEntrySourceBehavior behavior; private final int wordId; - public ReverseIndexPriorityEntrySource(BTreeReader reader, ReverseIndexEntrySourceBehavior behavior, int wordId) { + public ReverseIndexPriorityEntrySource(BTreeReader reader, int wordId) { this.reader = reader; - this.behavior = behavior; this.wordId = wordId; pos = 0; @@ -32,13 +29,6 @@ public class ReverseIndexPriorityEntrySource implements EntrySource { @Override public void read(LongQueryBuffer buffer) { - if (behavior == ReverseIndexEntrySourceBehavior.DO_NOT_PREFER - && buffer.hasRetainedData()) - { - pos = endOffset; - return; - } - buffer.end = min(buffer.end, endOffset - pos); reader.readData(buffer.data, buffer.end, pos); pos += buffer.end; diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityReader.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityReader.java index 9b553cb2..e314f0e5 100644 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityReader.java +++ b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/priority/ReverseIndexPriorityReader.java @@ -1,6 +1,5 @@ package nu.marginalia.index.priority; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; import nu.marginalia.index.query.EntrySource; import nu.marginalia.array.LongArray; import nu.marginalia.btree.BTreeReader; @@ -46,7 +45,7 @@ public class ReverseIndexPriorityReader { if (offset < 0) return new EmptyEntrySource(); - return new ReverseIndexPriorityEntrySource(createReaderNew(offset), ReverseIndexEntrySourceBehavior.DO_PREFER, wordId); + return new ReverseIndexPriorityEntrySource(createReaderNew(offset), wordId); } private BTreeReader createReaderNew(long offset) { diff --git a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexEntrySourceBehavior.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexEntrySourceBehavior.java deleted file mode 100644 index 4b84bda7..00000000 --- a/code/features-index/index-reverse/src/main/java/nu/marginalia/index/query/ReverseIndexEntrySourceBehavior.java +++ /dev/null @@ -1,11 +0,0 @@ -package nu.marginalia.index.query; - -public enum ReverseIndexEntrySourceBehavior { - /** Eagerly read from this entry source */ - DO_PREFER, - - /** Do not use this entry source if entries have been fetched - * from another entry source - */ - DO_NOT_PREFER -} diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest.java b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest.java index c8624196..a61f2a91 100644 --- a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest.java +++ b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest.java @@ -8,7 +8,6 @@ import nu.marginalia.index.journal.model.IndexJournalEntry; import nu.marginalia.index.journal.reader.IndexJournalReaderSingleCompressedFile; import nu.marginalia.index.journal.writer.IndexJournalWriterImpl; import nu.marginalia.index.journal.writer.IndexJournalWriter; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; import nu.marginalia.ranking.DomainRankings; import nu.marginalia.lexicon.KeywordLexicon; import nu.marginalia.lexicon.journal.KeywordLexiconJournal; @@ -101,22 +100,22 @@ class ReverseIndexFullConverterTest { System.out.println(reverseIndexReader.isWordInDoc(keywordLexicon.getReadOnly("2"), 3)); var buffer = new LongQueryBuffer(32); - reverseIndexReader.documents(keywordLexicon.getReadOnly("1"), ReverseIndexEntrySourceBehavior.DO_PREFER).read(buffer); + reverseIndexReader.documents(keywordLexicon.getReadOnly("1")).read(buffer); assertArrayEquals(LongStream.range(1, 17).map(v -> v | (255L << 32)).toArray(), buffer.copyData()); System.out.println(buffer); buffer.reset(); - reverseIndexReader.documents(keywordLexicon.getReadOnly("2"), ReverseIndexEntrySourceBehavior.DO_PREFER).read(buffer); + reverseIndexReader.documents(keywordLexicon.getReadOnly("2")).read(buffer); assertArrayEquals(LongStream.range(1, 17).map(v -> v*2).map(v -> v | (255L << 32)).toArray(), buffer.copyData()); System.out.println(buffer); buffer.reset(); - reverseIndexReader.documents(keywordLexicon.getReadOnly("3"), ReverseIndexEntrySourceBehavior.DO_PREFER).read(buffer); + reverseIndexReader.documents(keywordLexicon.getReadOnly("3")).read(buffer); assertArrayEquals(LongStream.range(1, 17).map(v -> v*3).map(v -> v | (255L << 32)).toArray(), buffer.copyData()); System.out.println(buffer); buffer.reset(); - var es = reverseIndexReader.documents(keywordLexicon.getReadOnly("7"), ReverseIndexEntrySourceBehavior.DO_PREFER); + var es = reverseIndexReader.documents(keywordLexicon.getReadOnly("7")); do { buffer.reset(); es.read(buffer); diff --git a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest2.java b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest2.java index fd6bb1d8..5ce603c1 100644 --- a/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest2.java +++ b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexFullConverterTest2.java @@ -10,7 +10,6 @@ import nu.marginalia.index.journal.reader.IndexJournalReaderSingleCompressedFile import nu.marginalia.index.journal.writer.IndexJournalWriterImpl; import nu.marginalia.index.journal.writer.IndexJournalWriter; import nu.marginalia.index.priority.ReverseIndexPriorityParameters; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; import nu.marginalia.ranking.DomainRankings; import nu.marginalia.lexicon.KeywordLexicon; import nu.marginalia.lexicon.journal.KeywordLexiconJournal; @@ -123,7 +122,7 @@ class ReverseIndexFullConverterTest2 { for (int i = workSetStart; i < workSetSize; i++) { - var es = reverseReader.documents(i, ReverseIndexEntrySourceBehavior.DO_PREFER); + var es = reverseReader.documents(i); LongQueryBuffer lqb = new LongQueryBuffer(100); while (es.hasMore()) { lqb.reset(); @@ -148,7 +147,7 @@ class ReverseIndexFullConverterTest2 { for (int i = workSetStart; i < workSetSize; i++) { - var es = reverseReader.documents(i, ReverseIndexEntrySourceBehavior.DO_PREFER); + var es = reverseReader.documents(i); LongQueryBuffer lqb = new LongQueryBuffer(100); while (es.hasMore()) { lqb.reset(); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java index 2656263a..709d2cd0 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java @@ -3,10 +3,7 @@ package nu.marginalia.index.index; import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.index.IndexServicesFactory; -import nu.marginalia.index.query.IndexQuery; -import nu.marginalia.index.query.IndexQueryBuilder; -import nu.marginalia.index.query.IndexQueryParams; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; +import nu.marginalia.index.query.*; import nu.marginalia.index.query.filter.QueryFilterStepFromPredicate; import nu.marginalia.index.svc.IndexSearchSetsService; import org.jetbrains.annotations.NotNull; @@ -15,7 +12,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.locks.Lock; @@ -102,32 +98,39 @@ public class SearchIndex { } final int[] orderedIncludes = terms.sortedDistinctIncludes(this::compareKeywords); + final int[] orderedIncludesPrio = terms.sortedDistinctIncludes(this::compareKeywordsPrio); List queryHeads = new ArrayList<>(10); List queries = new ArrayList<>(10); + // Fetch more results than specified for short queries, as the query itself is cheap and the + // priority index may contain a considerable amount of less interesting results + final int fetchSizeMultiplier; + if (orderedIncludes.length == 1) fetchSizeMultiplier = 4; + else fetchSizeMultiplier = 1; + // To ensure that good results are processed first, create query heads for the priority index that filter for terms // that contain pairs of two search terms - if (orderedIncludes.length > 1) { - for (int i = 0; i + 1 < orderedIncludes.length; i++) { - for (int j = i + 1; j < orderedIncludes.length; j++) { + if (orderedIncludesPrio.length > 1) { + for (int i = 0; i + 1 < orderedIncludesPrio.length; i++) { + for (int j = i + 1; j < orderedIncludesPrio.length; j++) { var entrySource = indexReader - .findPriorityWord(orderedIncludes[i]) - .alsoPrio(orderedIncludes[j]); + .findPriorityWord(IndexQueryPriority.BEST, orderedIncludesPrio[i], fetchSizeMultiplier) + .alsoPrio(orderedIncludesPrio[j]); queryHeads.add(entrySource); } } } // Next consider entries that appear only once in the priority index - for (var wordId : orderedIncludes) { - queryHeads.add(indexReader.findPriorityWord(wordId)); + for (var wordId : orderedIncludesPrio) { + queryHeads.add(indexReader.findPriorityWord(IndexQueryPriority.GOOD, wordId, fetchSizeMultiplier)); } // Finally consider terms in the full index, but only do this for sufficiently long queries // as short queries tend to be too underspecified to produce anything other than CPU warmth - if (orderedIncludes.length > 3) { - queryHeads.add(indexReader.findFullWord(orderedIncludes[0], ReverseIndexEntrySourceBehavior.DO_NOT_PREFER)); + if (orderedIncludes.length >= 3) { + queryHeads.add(indexReader.findFullWord(IndexQueryPriority.FALLBACK, orderedIncludes[0], fetchSizeMultiplier)); } for (var query : queryHeads) { @@ -162,6 +165,12 @@ public class SearchIndex { ); } + private int compareKeywordsPrio(int a, int b) { + return Long.compare( + indexReader.numHitsPrio(a), + indexReader.numHitsPrio(b) + ); + } /** Replaces the values of ids with their associated metadata, or 0L if absent */ public long[] getTermMetadata(int termId, long[] docs) { return indexReader.getMetadata(termId, docs); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java index dd6e4f17..7cb6f34f 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexQueryBuilder.java @@ -68,6 +68,6 @@ public class SearchIndexQueryBuilder implements IndexQueryBuilder { public IndexQuery build() { return query; - } + } } diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java index 60774ec2..db7bed61 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndexReader.java @@ -2,14 +2,10 @@ package nu.marginalia.index.index; import nu.marginalia.index.forward.ForwardIndexReader; import nu.marginalia.index.forward.ParamMatchingQueryFilter; -import nu.marginalia.index.query.EntrySource; -import nu.marginalia.index.query.IndexQuery; -import nu.marginalia.index.query.IndexQueryBuilder; -import nu.marginalia.index.query.IndexQueryParams; +import nu.marginalia.index.query.*; import nu.marginalia.index.query.filter.QueryFilterStepIf; import nu.marginalia.index.priority.ReverseIndexPriorityReader; import nu.marginalia.index.full.ReverseIndexFullReader; -import nu.marginalia.index.query.ReverseIndexEntrySourceBehavior; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,17 +27,21 @@ public class SearchIndexReader { this.reverseIndexPriorityReader = reverseIndexPriorityReader; } - public IndexQueryBuilder findPriorityWord(int wordId) { - return new SearchIndexQueryBuilder(reverseIndexFullReader, reverseIndexPriorityReader, new IndexQuery( - List.of(reverseIndexPriorityReader.priorityDocuments(wordId))), wordId); + public IndexQueryBuilder findPriorityWord(IndexQueryPriority priority, int wordId, int fetchSizeMultiplier) { + var sources = List.of(reverseIndexPriorityReader.priorityDocuments(wordId)); + + return new SearchIndexQueryBuilder(reverseIndexFullReader, reverseIndexPriorityReader, + new IndexQuery(sources, priority, fetchSizeMultiplier), wordId); } - public IndexQueryBuilder findFullWord(int wordId, ReverseIndexEntrySourceBehavior behavior) { - return new SearchIndexQueryBuilder(reverseIndexFullReader, reverseIndexPriorityReader, new IndexQuery( - List.of(reverseIndexFullReader.documents(wordId, behavior))), wordId); + public IndexQueryBuilder findFullWord(IndexQueryPriority priority, int wordId, int fetchSizeMultiplier) { + var sources = List.of(reverseIndexFullReader.documents(wordId)); + + return new SearchIndexQueryBuilder(reverseIndexFullReader, reverseIndexPriorityReader, + new IndexQuery(sources, priority, fetchSizeMultiplier), wordId); } - QueryFilterStepIf filterForParams(IndexQueryParams params) { + public QueryFilterStepIf filterForParams(IndexQueryParams params) { return new ParamMatchingQueryFilter(params, forwardIndexReader); } diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java index 13acaaf2..a480d3d6 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexQueryService.java @@ -17,6 +17,7 @@ import nu.marginalia.index.client.model.query.SearchSpecification; import nu.marginalia.array.buffer.LongQueryBuffer; import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.index.SearchIndexSearchTerms; +import nu.marginalia.index.query.IndexQueryPriority; import nu.marginalia.index.results.IndexMetadataService; import nu.marginalia.index.searchset.SearchSet; import nu.marginalia.index.results.IndexResultValuator; @@ -24,6 +25,7 @@ import nu.marginalia.index.query.IndexQuery; import nu.marginalia.index.results.IndexResultDomainDeduplicator; import nu.marginalia.index.svc.searchset.SmallSearchSet; import nu.marginalia.model.gson.GsonFactory; +import nu.marginalia.util.QueryParams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; @@ -152,13 +154,22 @@ public class IndexQueryService { private TLongList evaluateSubqueries(SearchParameters params) { final TLongList results = new TLongArrayList(params.fetchSize); - outer: // These queries are various term combinations for (var subquery : params.subqueries) { + if (!params.hasTimeLeft()) { + logger.info("Query timed out {}, ({}), -{}", + subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude); + break; + } + + logger.info(queryMarker, "{}", subquery); + final SearchIndexSearchTerms searchTerms = searchTermsSvc.getSearchTerms(subquery); + if (searchTerms.isEmpty()) { + logger.info(queryMarker, "empty"); continue; } @@ -167,26 +178,35 @@ public class IndexQueryService { // These queries are different indices for one subquery List queries = params.createIndexQueries(index, searchTerms); for (var query : queries) { - var resultsForSq = executeQuery(query, params, fetchSizeMultiplier(params, searchTerms)); + + if (!params.hasTimeLeft()) { + break; + } + + if (omitQuery(params, query, results.size())) { + logger.info(queryMarker, "Omitting {}", query); + continue; + } + + var resultsForSq = executeQuery(query, params); logger.info(queryMarker, "{} from {}", resultsForSq.size(), query); results.addAll(resultsForSq); - if (!params.hasTimeLeft()) { - logger.info("Query timed out {}, ({}), -{}", - subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude); - break outer; - } + } } return results; } - private int fetchSizeMultiplier(SearchParameters params, SearchIndexSearchTerms terms) { - if (terms.size() == 1) { - return 4; - } - return 1; + private boolean omitQuery(SearchParameters params, IndexQuery query, int resultCount) { + var priority = query.queryPriority; + + return switch (priority) { + case BEST -> false; + case GOOD -> resultCount > params.fetchSize / 4; + case FALLBACK -> resultCount != 0; + }; } private void logSearchTerms(SearchSubquery subquery, SearchIndexSearchTerms searchTerms) { @@ -214,9 +234,9 @@ public class IndexQueryService { } } - private TLongArrayList executeQuery(IndexQuery query, SearchParameters params, int fetchSizeMultiplier) + private TLongArrayList executeQuery(IndexQuery query, SearchParameters params) { - final int fetchSize = params.fetchSize * fetchSizeMultiplier; + final int fetchSize = params.fetchSize * query.fetchSizeMultiplier; final TLongArrayList results = new TLongArrayList(fetchSize); final LongQueryBuffer buffer = new LongQueryBuffer(fetchSize);