Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(index) Index optimization

Browse Source
This commit is contained in:
Viktor Lofgren 2024-02-25 14:01:58 +01:00
parent 3eb0800742
commit 1a51ec2d69
2 changed files with 31 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
code/index/java/nu/marginalia/index/IndexGrpcService.java
View File

@ -14,7 +14,10 @@ import nu.marginalia.api.searchquery.model.results.*;
import nu.marginalia.index.index.IndexQueryService; import nu.marginalia.index.index.IndexQueryService;
import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.index.index.StatefulIndex;
import nu.marginalia.index.model.SearchParameters; import nu.marginalia.index.model.SearchParameters;
import nu.marginalia.index.model.SearchTerms;
import nu.marginalia.index.model.SearchTermsUtil; import nu.marginalia.index.model.SearchTermsUtil;
import nu.marginalia.index.query.IndexQuery;
import nu.marginalia.index.query.IndexSearchBudget;
import nu.marginalia.index.results.IndexResultValuatorService; import nu.marginalia.index.results.IndexResultValuatorService;
import nu.marginalia.index.results.model.ids.CombinedDocIdList; import nu.marginalia.index.results.model.ids.CombinedDocIdList;
import nu.marginalia.index.searchset.SearchSetsService; import nu.marginalia.index.searchset.SearchSetsService;
@ -252,8 +255,15 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
/** Execute a search query */ /** Execute a search query */
public SearchResultSet run(SearchParameters parameters) throws SQLException, InterruptedException { public SearchResultSet run(SearchParameters parameters) throws SQLException, InterruptedException {
for (var subquery : parameters.subqueries) { for (var subquery : parameters.subqueries) {
workerPool.execute(new IndexLookup(subquery, parameters)); var terms = new SearchTerms(subquery);
if (terms.isEmpty())
continue;
for (var indexQuery : index.createQueries(terms, parameters.queryParams)) {
workerPool.execute(new IndexLookup(indexQuery, parameters.budget));
}
} }
for (int i = 0; i < indexValuationThreads; i++) { for (int i = 0; i < indexValuationThreads; i++) {
@ -283,12 +293,13 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
* resultCandidateQueue, which depending on the state of the valuator threads may * resultCandidateQueue, which depending on the state of the valuator threads may
* or may not block*/ * or may not block*/
class IndexLookup implements Runnable { class IndexLookup implements Runnable {
private final SearchSubquery subquery; private final IndexQuery query;
private final SearchParameters parameters; private final IndexSearchBudget budget;
IndexLookup(SearchSubquery subquery, SearchParameters parameters) { IndexLookup(IndexQuery query,
this.subquery = subquery; IndexSearchBudget budget) {
this.parameters = parameters; this.query = query;
this.budget = budget;
remainingIndexTasks.incrementAndGet(); remainingIndexTasks.incrementAndGet();
} }
@ -296,9 +307,8 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
public void run() { public void run() {
try { try {
indexQueryService.evaluateSubquery( indexQueryService.evaluateSubquery(
subquery, query,
parameters.queryParams, budget,
parameters.budget,
this::drain this::drain
); );
} }
@ -312,7 +322,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
} }
private void drain(CombinedDocIdList resultIds) { private void drain(CombinedDocIdList resultIds) {
long remainingTime = parameters.budget.timeLeft(); long remainingTime = budget.timeLeft();
try { try {
if (!resultCandidateQueue.offer(resultIds)) { if (!resultCandidateQueue.offer(resultIds)) {

34
code/index/java/nu/marginalia/index/index/IndexQueryService.java
View File

@ -16,7 +16,6 @@ import org.slf4j.LoggerFactory;
import org.slf4j.Marker; import org.slf4j.Marker;
import org.slf4j.MarkerFactory; import org.slf4j.MarkerFactory;
import java.util.List;
import java.util.function.Consumer; import java.util.function.Consumer;
@Singleton @Singleton
@ -35,36 +34,25 @@ public class IndexQueryService {
* at different priorty depths until timeout is reached or the results are all visited. * at different priorty depths until timeout is reached or the results are all visited.
* Then the results are combined. * Then the results are combined.
* */ * */
public void evaluateSubquery(SearchSubquery subquery, public void evaluateSubquery(IndexQuery query,
QueryParams queryParams,
IndexSearchBudget timeout, IndexSearchBudget timeout,
Consumer<CombinedDocIdList> drain) Consumer<CombinedDocIdList> drain)
{ {
final SearchTerms searchTerms = new SearchTerms(subquery); final LongArrayList results = new LongArrayList(512);
final Roaring64Bitmap results = new Roaring64Bitmap();
// These queries are different indices for one subquery // These queries are different indices for one subquery
List<IndexQuery> queries = index.createQueries(searchTerms, queryParams); final LongQueryBuffer buffer = new LongQueryBuffer(512);
for (var query : queries) {
if (!timeout.hasTimeLeft()) while (query.hasMore() && timeout.hasTimeLeft())
break; {
buffer.reset();
query.getMoreResults(buffer);
final LongQueryBuffer buffer = new LongQueryBuffer(512); results.addElements(0, buffer.data, 0, buffer.end);
while (query.hasMore() && timeout.hasTimeLeft()) if (results.size() < 512) {
{ drain.accept(new CombinedDocIdList(results));
buffer.reset(); results.clear();
query.getMoreResults(buffer);
for (int i = 0; i < buffer.size(); i++) {
results.add(buffer.data[i]);
}
if (results.getIntCardinality() > 512) {
drain.accept(new CombinedDocIdList(results));
results.clear();
}
} }
} }