Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(search-query) Add pagination to search query API and the direct query-service interface

Browse Source
This commit is contained in:
Viktor Lofgren 2024-09-25 12:56:38 +02:00
parent e9e8580913
commit 73f973cc06
5 changed files with 120 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
code/functions/search-query/api/src/main/protobuf/query-api.proto
View File

@ -30,6 +30,8 @@ message RpcQsQuery {
string searchSetIdentifier = 14;
string queryStrategy = 15; // Named query configuration
RpcTemporalBias temporalBias = 16;
RpcQsQueryPagination pagination = 17;
}
/* Query service query response */
@ -39,6 +41,19 @@ message RpcQsResponse {
repeated string searchTermsHuman = 3;
repeated string problems = 4;
string domain = 5;
RpcQsResultPagination pagination = 6;
}
message RpcQsQueryPagination {
int32 page = 1;
int32 pageSize = 2;
}
message RpcQsResultPagination {
int32 page = 1;
int32 pageSize = 2;
int32 totalResults = 3;
}
message RpcTemporalBias {

29
code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java
View File

@ -8,13 +8,13 @@ import io.prometheus.client.Histogram;
import nu.marginalia.api.searchquery.*;
import nu.marginalia.api.searchquery.model.query.ProcessedQuery;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.index.api.IndexClient;
import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.List;
@Singleton
public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase {
@ -54,12 +54,23 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase {
var indexRequest = QueryProtobufCodec.convertQuery(request, query);
var requestPagination = request.getPagination();
IndexClient.Pagination pagination = new IndexClient.Pagination(
requestPagination.getPage(),
requestPagination.getPageSize());
// Execute the query on the index partitions
List<RpcDecoratedResultItem> bestItems = indexClient.executeQueries(indexRequest);
IndexClient.AggregateQueryResponse response = indexClient.executeQueries(indexRequest, pagination);
// Convert results to response and send it back
var responseBuilder = RpcQsResponse.newBuilder()
.addAllResults(bestItems)
.addAllResults(response.results())
.setPagination(
RpcQsResultPagination.newBuilder()
.setPage(response.page())
.setTotalResults(response.totalResults())
)
.setSpecs(indexRequest)
.addAllSearchTermsHuman(query.searchTermsHuman);
@ -77,18 +88,22 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase {
}
public record DetailedDirectResult(ProcessedQuery processedQuery,
List<DecoratedSearchResultItem> result) {}
List<DecoratedSearchResultItem> result,
int totalResults) {}
/** Local query execution, without GRPC. */
public DetailedDirectResult executeDirect(
String originalQuery,
QueryParams params,
IndexClient.Pagination pagination,
ResultRankingParameters rankingParameters) {
var query = queryFactory.createQuery(params, rankingParameters);
var items = indexClient.executeQueries(QueryProtobufCodec.convertQuery(originalQuery, query));
IndexClient.AggregateQueryResponse response = indexClient.executeQueries(QueryProtobufCodec.convertQuery(originalQuery, query), pagination);
return new DetailedDirectResult(query, Lists.transform(items, QueryProtobufCodec::convertQueryResult));
return new DetailedDirectResult(query,
Lists.transform(response.results(), QueryProtobufCodec::convertQueryResult),
response.totalResults());
}
}

33
code/index/api/java/nu/marginalia/index/api/IndexClient.java
View File

@ -17,10 +17,14 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static java.lang.Math.clamp;
@Singleton
public class IndexClient {
private static final Logger logger = LoggerFactory.getLogger(IndexClient.class);
@ -39,17 +43,23 @@ public class IndexClient {
private static final Comparator<RpcDecoratedResultItem> comparator =
Comparator.comparing(RpcDecoratedResultItem::getRankingScore);
public record Pagination(int page, int pageSize) {}
public record AggregateQueryResponse(List<RpcDecoratedResultItem> results,
int page,
int totalResults
) {}
/** Execute a query on the index partitions and return the combined results. */
@SneakyThrows
public List<RpcDecoratedResultItem> executeQueries(RpcIndexQuery indexRequest) {
var futures =
public AggregateQueryResponse executeQueries(RpcIndexQuery indexRequest, Pagination pagination) {
List<CompletableFuture<Iterator<RpcDecoratedResultItem>>> futures =
channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query)
.async(executor)
.runEach(indexRequest);
final int resultsTotal = indexRequest.getQueryLimits().getResultsTotal();
final int resultsUpperBound = resultsTotal * channelPool.getNumNodes();
final int requestedMaxResults = indexRequest.getQueryLimits().getResultsTotal();
final int resultsUpperBound = requestedMaxResults * channelPool.getNumNodes();
List<RpcDecoratedResultItem> results = new ArrayList<>(resultsUpperBound);
@ -66,12 +76,17 @@ public class IndexClient {
results.sort(comparator);
results.removeIf(this::isBlacklisted);
// Keep only as many results as were requested
if (results.size() > resultsTotal) {
results = results.subList(0, resultsTotal);
}
int numReceivedResults = results.size();
return results;
// pagination is typically 1-indexed, so we need to adjust the start and end indices
int indexStart = (pagination.page - 1) * pagination.pageSize;
int indexEnd = (pagination.page) * pagination.pageSize;
results = results.subList(
clamp(indexStart, 0, results.size() - 1), // from is inclusive, so subtract 1 from size()
clamp(indexEnd, 0, results.size()));
return new AggregateQueryResponse(results, pagination.page(), numReceivedResults);
}
private boolean isBlacklisted(RpcDecoratedResultItem item) {

60
code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java
View File

@ -7,6 +7,7 @@ import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.functions.searchquery.QueryGRPCService;
import nu.marginalia.index.api.IndexClient;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.renderer.MustacheRenderer;
@ -15,8 +16,14 @@ import spark.Request;
import spark.Response;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static java.lang.Integer.min;
import static java.lang.Integer.parseInt;
import static java.util.Objects.requireNonNullElse;
public class QueryBasicInterface {
private final MustacheRenderer<Object> basicRenderer;
private final MustacheRenderer<Object> qdebugRenderer;
@ -34,38 +41,53 @@ public class QueryBasicInterface {
this.queryGRPCService = queryGRPCService;
}
/** Handle the basic search endpoint exposed in the bare-bones search interface. */
public Object handleBasic(Request request, Response response) {
String queryParams = request.queryParams("q");
if (queryParams == null) {
String queryString = request.queryParams("q");
if (queryString == null) {
return basicRenderer.render(new Object());
}
int count = request.queryParams("count") == null ? 10 : Integer.parseInt(request.queryParams("count"));
int domainCount = request.queryParams("domainCount") == null ? 5 : Integer.parseInt(request.queryParams("domainCount"));
String set = request.queryParams("set") == null ? "" : request.queryParams("set");
int count = parseInt(requireNonNullElse(request.queryParams("count"), "10"));
int page = parseInt(requireNonNullElse(request.queryParams("page"), "1"));
int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5"));
String set = requireNonNullElse(request.queryParams("set"), "");
var params = new QueryParams(queryParams, new QueryLimits(
domainCount, count, 250, 8192
var params = new QueryParams(queryString, new QueryLimits(
domainCount, min(100, count * 10), 250, 8192
), set);
var pagination = new IndexClient.Pagination(page, count);
var detailedDirectResult = queryGRPCService.executeDirect(
queryParams, params, ResultRankingParameters.sensibleDefaults()
queryString,
params,
pagination,
ResultRankingParameters.sensibleDefaults()
);
var results = detailedDirectResult.result();
List<PaginationInfoPage> paginationInfo = new ArrayList<>();
for (int i = 1; i <= detailedDirectResult.totalResults() / pagination.pageSize(); i++) {
paginationInfo.add(new PaginationInfoPage(i, i == pagination.page()));
}
if (request.headers("Accept").contains("application/json")) {
response.type("application/json");
return gson.toJson(results);
}
else {
return basicRenderer.render(
Map.of("query", queryParams,
Map.of("query", queryString,
"pages", paginationInfo,
"results", results)
);
}
}
/** Handle the qdebug endpoint, which allows for query debugging and ranking parameter tuning. */
public Object handleAdvanced(Request request, Response response) {
String queryString = request.queryParams("q");
if (queryString == null) {
@ -74,18 +96,24 @@ public class QueryBasicInterface {
);
}
int count = request.queryParams("count") == null ? 10 : Integer.parseInt(request.queryParams("count"));
int domainCount = request.queryParams("domainCount") == null ? 5 : Integer.parseInt(request.queryParams("domainCount"));
String set = request.queryParams("set") == null ? "" : request.queryParams("set");
int count = parseInt(requireNonNullElse(request.queryParams("count"), "10"));
int page = parseInt(requireNonNullElse(request.queryParams("page"), "1"));
int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5"));
String set = requireNonNullElse(request.queryParams("set"), "");
var queryParams = new QueryParams(queryString, new QueryLimits(
domainCount, count, 250, 8192
domainCount, min(100, count * 10), 250, 8192
), set);
var pagination = new IndexClient.Pagination(page, count);
var rankingParams = debugRankingParamsFromRequest(request);
var detailedDirectResult = queryGRPCService.executeDirect(
queryString, queryParams, rankingParams
queryString,
queryParams,
pagination,
rankingParams
);
var results = detailedDirectResult.result();
@ -127,10 +155,12 @@ public class QueryBasicInterface {
}
int intFromRequest(Request request, String param, int defaultValue) {
return Strings.isNullOrEmpty(request.queryParams(param)) ? defaultValue : Integer.parseInt(request.queryParams(param));
return Strings.isNullOrEmpty(request.queryParams(param)) ? defaultValue : parseInt(request.queryParams(param));
}
String stringFromRequest(Request request, String param, String defaultValue) {
return Strings.isNullOrEmpty(request.queryParams(param)) ? defaultValue : request.queryParams(param);
}
record PaginationInfoPage(int number, boolean current) {}
}

14
code/services-core/query-service/resources/templates/search.hdb
View File

@ -24,6 +24,20 @@
<div><small class="text-muted">{{url}}</small></div>
<p>{{description}}</p>
</div>
{{/each}}
<nav aria-label="pagination">
<ul class="pagination">
{{#each pages}}
<form action="/search">
<input type="hidden" name="q" value="{{query}}">
<input type="hidden" name="page" value="{{number}}">
<li class="page-item {{#if current}}active{{/if}}"><input type="submit" class="page-link" value="{{number}}"></li>
</form>
{{/each}}
</ul>
</nav>
{{#each pages}}
{{/each}}
{{/if}}
</div>