Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(index) Clean up

Browse Source
This commit is contained in:
Viktor Lofgren 2024-02-25 11:11:59 +01:00
parent f7f0100174
commit 9429bf5c45
3 changed files with 3 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/SearchResultItem.java
View File

@ -70,7 +70,7 @@ public class SearchResultItem implements Comparable<SearchResultItem> {
@Override @Override
public int compareTo(@NotNull SearchResultItem o) { public int compareTo(@NotNull SearchResultItem o) {
// this looks like a bug, but we actually want this in a reversed order // this looks like a bug, but we actually want this in a reversed order
int diff = Double.compare(o.getScore(), getScore()); int diff = Double.compare(getScore(), o.getScore());
if (diff != 0) if (diff != 0)
return diff; return diff;

7
code/index/java/nu/marginalia/index/IndexGrpcService.java
View File

@ -1,6 +1,5 @@
package nu.marginalia.index; package nu.marginalia.index;
import com.google.common.collect.MinMaxPriorityQueue;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.google.inject.Singleton; import com.google.inject.Singleton;
import io.grpc.stub.StreamObserver; import io.grpc.stub.StreamObserver;
@ -190,7 +189,7 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
ResultRankingContext rankingContext = createRankingContext(params.rankingParams, params.subqueries); ResultRankingContext rankingContext = createRankingContext(params.rankingParams, params.subqueries);
logger.info(queryMarker, "{}", params.queryParams); // logger.info(queryMarker, "{}", params.queryParams);
return new QueryExecution(rankingContext, params.fetchSize) return new QueryExecution(rankingContext, params.fetchSize)
.run(params); .run(params);
@ -243,8 +242,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
this.subquery = subquery; this.subquery = subquery;
this.parameters = parameters; this.parameters = parameters;
logger.info("Starting index task");
remainingIndexTasks.incrementAndGet(); remainingIndexTasks.incrementAndGet();
} }
@ -254,7 +251,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
subquery, subquery,
parameters.queryParams, parameters.queryParams,
parameters.budget, parameters.budget,
parameters.fetchSize,
this::drain this::drain
); );
} }
@ -264,7 +260,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
remainingIndexTasks.notifyAll(); remainingIndexTasks.notifyAll();
} }
} }
logger.info("Terminating index task");
} }
} }

60
code/index/java/nu/marginalia/index/index/IndexQueryService.java
View File

@ -38,28 +38,11 @@ public class IndexQueryService {
public void evaluateSubquery(SearchSubquery subquery, public void evaluateSubquery(SearchSubquery subquery,
QueryParams queryParams, QueryParams queryParams,
IndexSearchBudget timeout, IndexSearchBudget timeout,
int fetchSize,
Consumer<CombinedDocIdList> drain) Consumer<CombinedDocIdList> drain)
{ {
// These queries are various term combinations
if (!timeout.hasTimeLeft()) {
logger.info("Query timed out {}, ({}), -{}",
subquery.searchTermsInclude, subquery.searchTermsAdvice, subquery.searchTermsExclude);
return;
}
logger.info(queryMarker, "{}", subquery);
final SearchTerms searchTerms = new SearchTerms(subquery); final SearchTerms searchTerms = new SearchTerms(subquery);
if (searchTerms.isEmpty()) {
logger.info(queryMarker, "empty");
return;
}
final Roaring64Bitmap results = new Roaring64Bitmap(); final Roaring64Bitmap results = new Roaring64Bitmap();
// logSearchTerms(subquery, searchTerms);
// These queries are different indices for one subquery // These queries are different indices for one subquery
List<IndexQuery> queries = index.createQueries(searchTerms, queryParams); List<IndexQuery> queries = index.createQueries(searchTerms, queryParams);
for (var query : queries) { for (var query : queries) {
@ -69,9 +52,7 @@ public class IndexQueryService {
final LongQueryBuffer buffer = new LongQueryBuffer(512); final LongQueryBuffer buffer = new LongQueryBuffer(512);
while (query.hasMore() while (query.hasMore() && timeout.hasTimeLeft())
&& results.getIntCardinality() < fetchSize * query.fetchSizeMultiplier
&& timeout.hasTimeLeft())
{ {
buffer.reset(); buffer.reset();
query.getMoreResults(buffer); query.getMoreResults(buffer);
@ -85,8 +66,6 @@ public class IndexQueryService {
results.clear(); results.clear();
} }
} }
logger.info(queryMarker, "{} from {}", results.getIntCardinality(), query);
} }
if (!results.isEmpty()) { if (!results.isEmpty()) {
@ -94,41 +73,4 @@ public class IndexQueryService {
} }
} }
private void logSearchTerms(SearchSubquery subquery, SearchTerms searchTerms) {
// This logging should only be enabled in testing, as it is very verbose
// and contains sensitive information
if (!logger.isInfoEnabled(queryMarker)) {
return;
}
var includes = subquery.searchTermsInclude;
var advice = subquery.searchTermsAdvice;
var excludes = subquery.searchTermsExclude;
var priority = subquery.searchTermsPriority;
for (int i = 0; i < includes.size(); i++) {
logger.info(queryMarker, "{} -> {} I", includes.get(i),
Long.toHexString(searchTerms.includes().getLong(i))
);
}
for (int i = 0; i < advice.size(); i++) {
logger.info(queryMarker, "{} -> {} A", advice.get(i),
Long.toHexString(searchTerms.includes().getLong(includes.size() + i))
);
}
for (int i = 0; i < excludes.size(); i++) {
logger.info(queryMarker, "{} -> {} E", excludes.get(i),
Long.toHexString(searchTerms.excludes().getLong(i))
);
}
for (int i = 0; i < priority.size(); i++) {
logger.info(queryMarker, "{} -> {} P", priority.get(i),
Long.toHexString(searchTerms.priority().getLong(i))
);
}
}
} }