Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(refac) Remove ResultRankingParameters, QueryLimits class and use protobuf classes directly instead

Browse Source 
This is primarily to make the code a bit easier to reason about, and will reduce the level of indirection and data copying in the search-servi->query-service->index-service communication chain.
This commit is contained in:
Viktor Lofgren 2025-01-08 16:15:57 +01:00
parent b7f0a2a98e
commit a1fb92468f
33 changed files with 343 additions and 605 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java
View File

@ -2,9 +2,6 @@ package nu.marginalia.api.searchquery;
import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.index.query.limit.SpecificationLimitType;
@ -27,37 +24,19 @@ public class IndexProtobufCodec {
.build();
}
public static QueryLimits convertQueryLimits(RpcQueryLimits queryLimits) {
return new QueryLimits(
queryLimits.getResultsByDomain(),
queryLimits.getResultsTotal(),
queryLimits.getTimeoutMs(),
queryLimits.getFetchSize()
);
}
public static RpcQueryLimits convertQueryLimits(QueryLimits queryLimits) {
return RpcQueryLimits.newBuilder()
.setResultsByDomain(queryLimits.resultsByDomain())
.setResultsTotal(queryLimits.resultsTotal())
.setTimeoutMs(queryLimits.timeoutMs())
.setFetchSize(queryLimits.fetchSize())
.build();
}
public static SearchQuery convertRpcQuery(RpcQuery query) {
List<SearchPhraseConstraint> phraeConstraints = new ArrayList<>();
List<SearchPhraseConstraint> phraseConstraints = new ArrayList<>();
for (int j = 0; j < query.getPhrasesCount(); j++) {
var coh = query.getPhrases(j);
if (coh.getType() == RpcPhrases.TYPE.OPTIONAL) {
phraeConstraints.add(new SearchPhraseConstraint.Optional(List.copyOf(coh.getTermsList())));
phraseConstraints.add(new SearchPhraseConstraint.Optional(List.copyOf(coh.getTermsList())));
}
else if (coh.getType() == RpcPhrases.TYPE.MANDATORY) {
phraeConstraints.add(new SearchPhraseConstraint.Mandatory(List.copyOf(coh.getTermsList())));
phraseConstraints.add(new SearchPhraseConstraint.Mandatory(List.copyOf(coh.getTermsList())));
}
else if (coh.getType() == RpcPhrases.TYPE.FULL) {
phraeConstraints.add(new SearchPhraseConstraint.Full(List.copyOf(coh.getTermsList())));
phraseConstraints.add(new SearchPhraseConstraint.Full(List.copyOf(coh.getTermsList())));
}
else {
throw new IllegalArgumentException("Unknown phrase constraint type: " + coh.getType());
@ -70,7 +49,7 @@ public class IndexProtobufCodec {
query.getExcludeList(),
query.getAdviceList(),
query.getPriorityList(),
phraeConstraints
phraseConstraints
);
}
@ -103,62 +82,4 @@ public class IndexProtobufCodec {
return subqueryBuilder.build();
}
public static ResultRankingParameters convertRankingParameterss(RpcResultRankingParameters params) {
if (params == null)
return ResultRankingParameters.sensibleDefaults();
return new ResultRankingParameters(
new Bm25Parameters(params.getBm25K(), params.getBm25B()),
params.getShortDocumentThreshold(),
params.getShortDocumentPenalty(),
params.getDomainRankBonus(),
params.getQualityPenalty(),
params.getShortSentenceThreshold(),
params.getShortSentencePenalty(),
params.getBm25Weight(),
params.getTcfFirstPositionWeight(),
params.getTcfVerbatimWeight(),
params.getTcfProximityWeight(),
ResultRankingParameters.TemporalBias.valueOf(params.getTemporalBias().getBias().name()),
params.getTemporalBiasWeight(),
params.getDisablePenalties(),
params.getExportDebugData()
);
}
public static RpcResultRankingParameters convertRankingParameterss(ResultRankingParameters rankingParams,
RpcTemporalBias temporalBias)
{
if (rankingParams == null) {
rankingParams = ResultRankingParameters.sensibleDefaults();
}
var builder = RpcResultRankingParameters.newBuilder()
.setBm25B(rankingParams.bm25Params.b())
.setBm25K(rankingParams.bm25Params.k())
.setShortDocumentThreshold(rankingParams.shortDocumentThreshold)
.setShortDocumentPenalty(rankingParams.shortDocumentPenalty)
.setDomainRankBonus(rankingParams.domainRankBonus)
.setQualityPenalty(rankingParams.qualityPenalty)
.setShortSentenceThreshold(rankingParams.shortSentenceThreshold)
.setShortSentencePenalty(rankingParams.shortSentencePenalty)
.setBm25Weight(rankingParams.bm25Weight)
.setTcfFirstPositionWeight(rankingParams.tcfFirstPosition)
.setTcfProximityWeight(rankingParams.tcfProximity)
.setTcfVerbatimWeight(rankingParams.tcfVerbatim)
.setTemporalBiasWeight(rankingParams.temporalBiasWeight)
.setDisablePenalties(rankingParams.disablePenalties)
.setExportDebugData(rankingParams.exportDebugData);
if (temporalBias != null && temporalBias.getBias() != RpcTemporalBias.Bias.NONE) {
builder.setTemporalBias(temporalBias);
}
else {
builder.setTemporalBias(RpcTemporalBias.newBuilder()
.setBias(RpcTemporalBias.Bias.valueOf(rankingParams.temporalBias.name())));
}
return builder.build();
}
}

38
code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java
View File

@ -5,7 +5,7 @@ import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.query.QueryResponse;
import nu.marginalia.api.searchquery.model.query.SearchSpecification;
import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.api.searchquery.model.results.SearchResultItem;
import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore;
import nu.marginalia.api.searchquery.model.results.debug.DebugFactor;
@ -15,10 +15,7 @@ import nu.marginalia.api.searchquery.model.results.debug.ResultRankingDetails;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.model.EdgeUrl;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
public class QueryProtobufCodec {
@ -37,7 +34,7 @@ public class QueryProtobufCodec {
builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size));
builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank));
builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits));
builder.setQueryLimits(query.specs.queryLimits);
// Query strategy may be overridden by the query, but if not, use the one from the request
if (query.specs.queryStrategy != null && query.specs.queryStrategy != QueryStrategy.AUTO)
@ -45,8 +42,12 @@ public class QueryProtobufCodec {
else
builder.setQueryStrategy(request.getQueryStrategy());
if (query.specs.rankingParams != null) {
builder.setParameters(IndexProtobufCodec.convertRankingParameterss(query.specs.rankingParams, request.getTemporalBias()));
if (query.specs.rankingParams != null && request.getTemporalBias().getBias() != RpcTemporalBias.Bias.NONE) {
builder.setParameters(
RpcResultRankingParameters.newBuilder(query.specs.rankingParams)
.setTemporalBias(request.getTemporalBias())
.build()
);
}
return builder.build();
@ -65,18 +66,13 @@ public class QueryProtobufCodec {
builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size));
builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank));
builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits));
builder.setQueryLimits(query.specs.queryLimits);
// Query strategy may be overridden by the query, but if not, use the one from the request
builder.setQueryStrategy(query.specs.queryStrategy.name());
if (query.specs.rankingParams != null) {
builder.setParameters(IndexProtobufCodec.convertRankingParameterss(
query.specs.rankingParams,
RpcTemporalBias.newBuilder().setBias(
RpcTemporalBias.Bias.NONE)
.build())
);
builder.setParameters(query.specs.rankingParams);
}
return builder.build();
@ -95,10 +91,10 @@ public class QueryProtobufCodec {
IndexProtobufCodec.convertSpecLimit(request.getSize()),
IndexProtobufCodec.convertSpecLimit(request.getRank()),
request.getDomainIdsList(),
IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()),
request.getQueryLimits(),
request.getSearchSetIdentifier(),
QueryStrategy.valueOf(request.getQueryStrategy()),
ResultRankingParameters.TemporalBias.valueOf(request.getTemporalBias().getBias().name()),
RpcTemporalBias.Bias.valueOf(request.getTemporalBias().getBias().name()),
request.getPagination().getPage()
);
}
@ -294,9 +290,9 @@ public class QueryProtobufCodec {
IndexProtobufCodec.convertSpecLimit(specs.getYear()),
IndexProtobufCodec.convertSpecLimit(specs.getSize()),
IndexProtobufCodec.convertSpecLimit(specs.getRank()),
IndexProtobufCodec.convertQueryLimits(specs.getQueryLimits()),
specs.getQueryLimits(),
QueryStrategy.valueOf(specs.getQueryStrategy()),
IndexProtobufCodec.convertRankingParameterss(specs.getParameters())
Objects.requireNonNullElseGet(specs.getParameters(), PrototypeRankingParameters::sensibleDefaults)
);
}
@ -307,7 +303,7 @@ public class QueryProtobufCodec {
.addAllTacitExcludes(params.tacitExcludes())
.addAllTacitPriority(params.tacitPriority())
.setHumanQuery(params.humanQuery())
.setQueryLimits(IndexProtobufCodec.convertQueryLimits(params.limits()))
.setQueryLimits(params.limits())
.setQuality(IndexProtobufCodec.convertSpecLimit(params.quality()))
.setYear(IndexProtobufCodec.convertSpecLimit(params.year()))
.setSize(IndexProtobufCodec.convertSpecLimit(params.size()))
@ -319,7 +315,7 @@ public class QueryProtobufCodec {
.build())
.setPagination(RpcQsQueryPagination.newBuilder()
.setPage(params.page())
.setPageSize(Math.min(100, params.limits().resultsTotal()))
.setPageSize(Math.min(100, params.limits().getResultsTotal()))
.build());
if (params.nearDomain() != null)

12
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.api.searchquery.model.query;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
@ -21,14 +21,14 @@ public record QueryParams(
SpecificationLimit size,
SpecificationLimit rank,
List<Integer> domainIds,
QueryLimits limits,
RpcQueryLimits limits,
String identifier,
QueryStrategy queryStrategy,
ResultRankingParameters.TemporalBias temporalBias,
RpcTemporalBias.Bias temporalBias,
int page
)
{
public QueryParams(String query, QueryLimits limits, String identifier) {
public QueryParams(String query, RpcQueryLimits limits, String identifier) {
this(query, null,
List.of(),
List.of(),
@ -42,7 +42,7 @@ public record QueryParams(
limits,
identifier,
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
1 // page
);
}

24
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.api.searchquery.model.query;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
@ -24,11 +24,11 @@ public class SearchSpecification {
public SpecificationLimit size;
public SpecificationLimit rank;
public final QueryLimits queryLimits;
public final RpcQueryLimits queryLimits;
public final QueryStrategy queryStrategy;
public final ResultRankingParameters rankingParams;
public final RpcResultRankingParameters rankingParams;
public SearchSpecification(SearchQuery query,
List<Integer> domains,
@ -38,9 +38,9 @@ public class SearchSpecification {
SpecificationLimit year,
SpecificationLimit size,
SpecificationLimit rank,
QueryLimits queryLimits,
RpcQueryLimits queryLimits,
QueryStrategy queryStrategy,
ResultRankingParameters rankingParams)
RpcResultRankingParameters rankingParams)
{
this.query = query;
this.domains = domains;
@ -91,7 +91,7 @@ public class SearchSpecification {
return this.rank;
}
public QueryLimits getQueryLimits() {
public RpcQueryLimits getQueryLimits() {
return this.queryLimits;
}
@ -99,7 +99,7 @@ public class SearchSpecification {
return this.queryStrategy;
}
public ResultRankingParameters getRankingParams() {
public RpcResultRankingParameters getRankingParams() {
return this.rankingParams;
}
@ -120,9 +120,9 @@ public class SearchSpecification {
private boolean size$set;
private SpecificationLimit rank$value;
private boolean rank$set;
private QueryLimits queryLimits;
private RpcQueryLimits queryLimits;
private QueryStrategy queryStrategy;
private ResultRankingParameters rankingParams;
private RpcResultRankingParameters rankingParams;
SearchSpecificationBuilder() {
}
@ -171,7 +171,7 @@ public class SearchSpecification {
return this;
}
public SearchSpecificationBuilder queryLimits(QueryLimits queryLimits) {
public SearchSpecificationBuilder queryLimits(RpcQueryLimits queryLimits) {
this.queryLimits = queryLimits;
return this;
}
@ -181,7 +181,7 @@ public class SearchSpecification {
return this;
}
public SearchSpecificationBuilder rankingParams(ResultRankingParameters rankingParams) {
public SearchSpecificationBuilder rankingParams(RpcResultRankingParameters rankingParams) {
this.rankingParams = rankingParams;
return this;
}

33
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/PrototypeRankingParameters.java Normal file
View File

@ -0,0 +1,33 @@
package nu.marginalia.api.searchquery.model.results;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.RpcTemporalBias;
public class PrototypeRankingParameters {
/** These are the default ranking parameters that are used when no parameters are specified. */
private static final RpcResultRankingParameters _sensibleDefaults = RpcResultRankingParameters.newBuilder()
.setBm25B(0.5)
.setBm25K(1.2)
.setShortDocumentThreshold(2000)
.setShortDocumentPenalty(2.)
.setDomainRankBonus(1 / 100.)
.setQualityPenalty(1 / 15.)
.setShortSentenceThreshold(2)
.setShortSentencePenalty(5)
.setBm25Weight(1.)
.setTcfVerbatimWeight(1.)
.setTcfProximityWeight(1.)
.setTcfFirstPositionWeight(5)
.setTemporalBias(RpcTemporalBias.newBuilder().setBias(RpcTemporalBias.Bias.NONE))
.setTemporalBiasWeight(5.0)
.setExportDebugData(false)
.setDisablePenalties(false)
.build();
public static RpcResultRankingParameters sensibleDefaults() {
return _sensibleDefaults;
}
}

5
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java
View File

@ -1,12 +1,13 @@
package nu.marginalia.api.searchquery.model.results;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
import java.util.BitSet;
public class ResultRankingContext {
private final int docCount;
public final ResultRankingParameters params;
public final RpcResultRankingParameters params;
public final BitSet regularMask;
@ -21,7 +22,7 @@ public class ResultRankingContext {
public final CqDataInt priorityCounts;
public ResultRankingContext(int docCount,
ResultRankingParameters params,
RpcResultRankingParameters params,
BitSet ngramsMask,
BitSet regularMask,
CqDataInt fullCounts,

291
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java
View File

@ -1,291 +0,0 @@
package nu.marginalia.api.searchquery.model.results;
import java.util.Objects;
public class ResultRankingParameters {
/**
* Tuning for BM25 when applied to full document matches
*/
public final Bm25Parameters bm25Params;
/**
* Documents below this length are penalized
*/
public int shortDocumentThreshold;
public double shortDocumentPenalty;
/**
* Scaling factor associated with domain rank (unscaled rank value is 0-255; high is good)
*/
public double domainRankBonus;
/**
* Scaling factor associated with document quality (unscaled rank value is 0-15; high is bad)
*/
public double qualityPenalty;
/**
* Average sentence length values below this threshold are penalized, range [0-4), 2 or 3 is probably what you want
*/
public int shortSentenceThreshold;
/**
* Magnitude of penalty for documents with low average sentence length
*/
public double shortSentencePenalty;
public double bm25Weight;
public double tcfFirstPosition;
public double tcfVerbatim;
public double tcfProximity;
public TemporalBias temporalBias;
public double temporalBiasWeight;
public boolean disablePenalties;
public boolean exportDebugData;
public ResultRankingParameters(Bm25Parameters bm25Params, int shortDocumentThreshold, double shortDocumentPenalty, double domainRankBonus, double qualityPenalty, int shortSentenceThreshold, double shortSentencePenalty, double bm25Weight, double tcfFirstPosition, double tcfVerbatim, double tcfProximity, TemporalBias temporalBias, double temporalBiasWeight, boolean disablePenalties, boolean exportDebugData) {
this.bm25Params = bm25Params;
this.shortDocumentThreshold = shortDocumentThreshold;
this.shortDocumentPenalty = shortDocumentPenalty;
this.domainRankBonus = domainRankBonus;
this.qualityPenalty = qualityPenalty;
this.shortSentenceThreshold = shortSentenceThreshold;
this.shortSentencePenalty = shortSentencePenalty;
this.bm25Weight = bm25Weight;
this.tcfFirstPosition = tcfFirstPosition;
this.tcfVerbatim = tcfVerbatim;
this.tcfProximity = tcfProximity;
this.temporalBias = temporalBias;
this.temporalBiasWeight = temporalBiasWeight;
this.disablePenalties = disablePenalties;
this.exportDebugData = exportDebugData;
}
private static final ResultRankingParameters _sensibleDefaults = builder()
.bm25Params(new Bm25Parameters(1.2, 0.5))
.shortDocumentThreshold(2000)
.shortDocumentPenalty(2.)
.domainRankBonus(1 / 100.)
.qualityPenalty(1 / 15.)
.shortSentenceThreshold(2)
.shortSentencePenalty(5)
.bm25Weight(1.)
.tcfVerbatim(1.)
.tcfProximity(1.)
.tcfFirstPosition(5)
.temporalBias(TemporalBias.NONE)
.temporalBiasWeight(5.0)
.exportDebugData(false)
.disablePenalties(false)
.build();
public static ResultRankingParameters sensibleDefaults() {
return _sensibleDefaults;
}
public static ResultRankingParametersBuilder builder() {
return new ResultRankingParametersBuilder();
}
public Bm25Parameters getBm25Params() {
return this.bm25Params;
}
public int getShortDocumentThreshold() {
return this.shortDocumentThreshold;
}
public double getShortDocumentPenalty() {
return this.shortDocumentPenalty;
}
public double getDomainRankBonus() {
return this.domainRankBonus;
}
public double getQualityPenalty() {
return this.qualityPenalty;
}
public int getShortSentenceThreshold() {
return this.shortSentenceThreshold;
}
public double getShortSentencePenalty() {
return this.shortSentencePenalty;
}
public double getBm25Weight() {
return this.bm25Weight;
}
public double getTcfFirstPosition() {
return this.tcfFirstPosition;
}
public double getTcfVerbatim() {
return this.tcfVerbatim;
}
public double getTcfProximity() {
return this.tcfProximity;
}
public TemporalBias getTemporalBias() {
return this.temporalBias;
}
public double getTemporalBiasWeight() {
return this.temporalBiasWeight;
}
public boolean isDisablePenalties() { return this.disablePenalties; }
public boolean isExportDebugData() {
return this.exportDebugData;
}
@Override
public final boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof ResultRankingParameters that)) return false;
return shortDocumentThreshold == that.shortDocumentThreshold && Double.compare(shortDocumentPenalty, that.shortDocumentPenalty) == 0 && Double.compare(domainRankBonus, that.domainRankBonus) == 0 && Double.compare(qualityPenalty, that.qualityPenalty) == 0 && shortSentenceThreshold == that.shortSentenceThreshold && Double.compare(shortSentencePenalty, that.shortSentencePenalty) == 0 && Double.compare(bm25Weight, that.bm25Weight) == 0 && Double.compare(tcfFirstPosition, that.tcfFirstPosition) == 0 && Double.compare(tcfVerbatim, that.tcfVerbatim) == 0 && Double.compare(tcfProximity, that.tcfProximity) == 0 && Double.compare(temporalBiasWeight, that.temporalBiasWeight) == 0 && exportDebugData == that.exportDebugData && Objects.equals(bm25Params, that.bm25Params) && temporalBias == that.temporalBias;
}
@Override
public int hashCode() {
int result = Objects.hashCode(bm25Params);
result = 31 * result + shortDocumentThreshold;
result = 31 * result + Double.hashCode(shortDocumentPenalty);
result = 31 * result + Double.hashCode(domainRankBonus);
result = 31 * result + Double.hashCode(qualityPenalty);
result = 31 * result + shortSentenceThreshold;
result = 31 * result + Double.hashCode(shortSentencePenalty);
result = 31 * result + Double.hashCode(bm25Weight);
result = 31 * result + Double.hashCode(tcfFirstPosition);
result = 31 * result + Double.hashCode(tcfVerbatim);
result = 31 * result + Double.hashCode(tcfProximity);
result = 31 * result + Objects.hashCode(temporalBias);
result = 31 * result + Double.hashCode(temporalBiasWeight);
result = 31 * result + Boolean.hashCode(disablePenalties);
result = 31 * result + Boolean.hashCode(exportDebugData);
return result;
}
public String toString() {
return "ResultRankingParameters(bm25Params=" + this.getBm25Params() + ", shortDocumentThreshold=" + this.getShortDocumentThreshold() + ", shortDocumentPenalty=" + this.getShortDocumentPenalty() + ", domainRankBonus=" + this.getDomainRankBonus() + ", qualityPenalty=" + this.getQualityPenalty() + ", shortSentenceThreshold=" + this.getShortSentenceThreshold() + ", shortSentencePenalty=" + this.getShortSentencePenalty() + ", bm25Weight=" + this.getBm25Weight() + ", tcfFirstPosition=" + this.getTcfFirstPosition() + ", tcfVerbatim=" + this.getTcfVerbatim() + ", tcfProximity=" + this.getTcfProximity() + ", temporalBias=" + this.getTemporalBias() + ", temporalBiasWeight=" + this.getTemporalBiasWeight() + ", exportDebugData=" + this.isExportDebugData() + ")";
}
public enum TemporalBias {
RECENT, OLD, NONE
}
public static class ResultRankingParametersBuilder {
private Bm25Parameters bm25Params;
private int shortDocumentThreshold;
private double shortDocumentPenalty;
private double domainRankBonus;
private double qualityPenalty;
private int shortSentenceThreshold;
private double shortSentencePenalty;
private double bm25Weight;
private double tcfFirstPosition;
private double tcfVerbatim;
private double tcfProximity;
private TemporalBias temporalBias;
private double temporalBiasWeight;
private boolean disablePenalties;
private boolean exportDebugData;
ResultRankingParametersBuilder() {
}
public ResultRankingParametersBuilder bm25Params(Bm25Parameters bm25Params) {
this.bm25Params = bm25Params;
return this;
}
public ResultRankingParametersBuilder shortDocumentThreshold(int shortDocumentThreshold) {
this.shortDocumentThreshold = shortDocumentThreshold;
return this;
}
public ResultRankingParametersBuilder shortDocumentPenalty(double shortDocumentPenalty) {
this.shortDocumentPenalty = shortDocumentPenalty;
return this;
}
public ResultRankingParametersBuilder domainRankBonus(double domainRankBonus) {
this.domainRankBonus = domainRankBonus;
return this;
}
public ResultRankingParametersBuilder qualityPenalty(double qualityPenalty) {
this.qualityPenalty = qualityPenalty;
return this;
}
public ResultRankingParametersBuilder shortSentenceThreshold(int shortSentenceThreshold) {
this.shortSentenceThreshold = shortSentenceThreshold;
return this;
}
public ResultRankingParametersBuilder shortSentencePenalty(double shortSentencePenalty) {
this.shortSentencePenalty = shortSentencePenalty;
return this;
}
public ResultRankingParametersBuilder bm25Weight(double bm25Weight) {
this.bm25Weight = bm25Weight;
return this;
}
public ResultRankingParametersBuilder tcfFirstPosition(double tcfFirstPosition) {
this.tcfFirstPosition = tcfFirstPosition;
return this;
}
public ResultRankingParametersBuilder tcfVerbatim(double tcfVerbatim) {
this.tcfVerbatim = tcfVerbatim;
return this;
}
public ResultRankingParametersBuilder tcfProximity(double tcfProximity) {
this.tcfProximity = tcfProximity;
return this;
}
public ResultRankingParametersBuilder temporalBias(TemporalBias temporalBias) {
this.temporalBias = temporalBias;
return this;
}
public ResultRankingParametersBuilder temporalBiasWeight(double temporalBiasWeight) {
this.temporalBiasWeight = temporalBiasWeight;
return this;
}
public ResultRankingParametersBuilder disablePenalties(boolean disablePenalties) {
this.disablePenalties = disablePenalties;
return this;
}
public ResultRankingParametersBuilder exportDebugData(boolean exportDebugData) {
this.exportDebugData = exportDebugData;
return this;
}
public ResultRankingParameters build() {
return new ResultRankingParameters(this.bm25Params, this.shortDocumentThreshold, this.shortDocumentPenalty, this.domainRankBonus, this.qualityPenalty, this.shortSentenceThreshold, this.shortSentencePenalty, this.bm25Weight, this.tcfFirstPosition, this.tcfVerbatim, this.tcfProximity, this.temporalBias, this.temporalBiasWeight, this.disablePenalties, this.exportDebugData);
}
}
}

14
code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java
View File

@ -3,8 +3,6 @@ package nu.marginalia.index.client;
import nu.marginalia.api.searchquery.IndexProtobufCodec;
import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.SpecificationLimit;
import org.junit.jupiter.api.Test;
@ -22,18 +20,6 @@ class IndexProtobufCodecTest {
verifyIsIdentityTransformation(SpecificationLimit.lessThan(1), l -> IndexProtobufCodec.convertSpecLimit(IndexProtobufCodec.convertSpecLimit(l)));
}
@Test
public void testRankingParameters() {
verifyIsIdentityTransformation(ResultRankingParameters.sensibleDefaults(),
p -> IndexProtobufCodec.convertRankingParameterss(IndexProtobufCodec.convertRankingParameterss(p, null)));
}
@Test
public void testQueryLimits() {
verifyIsIdentityTransformation(new QueryLimits(1,2,3,4),
l -> IndexProtobufCodec.convertQueryLimits(IndexProtobufCodec.convertQueryLimits(l))
);
}
@Test
public void testSubqery() {
verifyIsIdentityTransformation(new SearchQuery(

9
code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryFactory.java
View File

@ -2,8 +2,9 @@ package nu.marginalia.functions.searchquery;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.model.query.*;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.functions.searchquery.query_parser.QueryExpansion;
import nu.marginalia.functions.searchquery.query_parser.QueryParser;
import nu.marginalia.functions.searchquery.query_parser.token.QueryToken;
@ -36,7 +37,7 @@ public class QueryFactory {
public ProcessedQuery createQuery(QueryParams params,
@Nullable ResultRankingParameters rankingParams) {
@Nullable RpcResultRankingParameters rankingParams) {
final var query = params.humanQuery();
if (query.length() > 1000) {
@ -132,7 +133,9 @@ public class QueryFactory {
var limits = params.limits();
// Disable limits on number of results per domain if we're searching with a site:-type term
if (domain != null) {
limits = limits.forSingleDomain();
limits = RpcQueryLimits.newBuilder(limits)
.setResultsByDomain(limits.getResultsTotal())
.build();
}
var expansion = queryExpansion.expandQuery(queryBuilder.searchTermsInclude);

6
code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java
View File

@ -9,7 +9,7 @@ import nu.marginalia.api.searchquery.*;
import nu.marginalia.api.searchquery.model.query.ProcessedQuery;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.index.api.IndexClient;
import nu.marginalia.service.server.DiscoverableService;
import org.slf4j.Logger;
@ -55,7 +55,7 @@ public class QueryGRPCService
.time(() -> {
var params = QueryProtobufCodec.convertRequest(request);
var query = queryFactory.createQuery(params, ResultRankingParameters.sensibleDefaults());
var query = queryFactory.createQuery(params, PrototypeRankingParameters.sensibleDefaults());
var indexRequest = QueryProtobufCodec.convertQuery(request, query);
@ -102,7 +102,7 @@ public class QueryGRPCService
String originalQuery,
QueryParams params,
IndexClient.Pagination pagination,
ResultRankingParameters rankingParameters) {
RpcResultRankingParameters rankingParameters) {
var query = queryFactory.createQuery(params, rankingParameters);
IndexClient.AggregateQueryResponse response = indexClient.executeQueries(QueryProtobufCodec.convertQuery(originalQuery, query), pagination);

13
code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java
View File

@ -1,12 +1,12 @@
package nu.marginalia.query.svc;
import nu.marginalia.WmsaHome;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.query.SearchSpecification;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.functions.searchquery.QueryFactory;
import nu.marginalia.functions.searchquery.query_parser.QueryExpansion;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.index.query.limit.SpecificationLimitType;
@ -49,10 +49,15 @@ public class QueryFactoryTest {
SpecificationLimit.none(),
SpecificationLimit.none(),
null,
new QueryLimits(100, 100, 100, 100),
RpcQueryLimits.newBuilder()
.setResultsTotal(100)
.setResultsByDomain(100)
.setTimeoutMs(100)
.setFetchSize(100)
.build(),
"NONE",
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
0), null).specs;
}

4
code/index/java/nu/marginalia/index/IndexGrpcService.java
View File

@ -10,12 +10,12 @@ import it.unimi.dsi.fastutil.longs.LongArrayList;
import nu.marginalia.api.searchquery.IndexApiGrpc;
import nu.marginalia.api.searchquery.RpcDecoratedResultItem;
import nu.marginalia.api.searchquery.RpcIndexQuery;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.model.compiled.CompiledQuery;
import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong;
import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
import nu.marginalia.api.searchquery.model.query.SearchSpecification;
import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.array.page.LongQueryBuffer;
import nu.marginalia.index.index.StatefulIndex;
import nu.marginalia.index.model.SearchParameters;
@ -211,7 +211,7 @@ public class IndexGrpcService
/** This class is responsible for ranking the results and adding the best results to the
* resultHeap, which depending on the state of the indexLookup threads may or may not block
*/
private ResultRankingContext createRankingContext(ResultRankingParameters rankingParams,
private ResultRankingContext createRankingContext(RpcResultRankingParameters rankingParams,
CompiledQuery<String> compiledQuery,
CompiledQueryLong compiledQueryIds)
{

29
code/index/java/nu/marginalia/index/model/SearchParameters.java
View File

@ -2,16 +2,19 @@ package nu.marginalia.index.model;
import nu.marginalia.api.searchquery.IndexProtobufCodec;
import nu.marginalia.api.searchquery.RpcIndexQuery;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.model.compiled.CompiledQuery;
import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong;
import nu.marginalia.api.searchquery.model.compiled.CompiledQueryParser;
import nu.marginalia.api.searchquery.model.query.SearchSpecification;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.query.SearchSpecification;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.index.query.IndexSearchBudget;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.searchset.SearchSet;
import java.util.Objects;
import static nu.marginalia.api.searchquery.IndexProtobufCodec.convertSpecLimit;
public class SearchParameters {
@ -23,7 +26,7 @@ public class SearchParameters {
public final IndexSearchBudget budget;
public final SearchQuery query;
public final QueryParams queryParams;
public final ResultRankingParameters rankingParams;
public final RpcResultRankingParameters rankingParams;
public final int limitByDomain;
public final int limitTotal;
@ -41,11 +44,11 @@ public class SearchParameters {
public SearchParameters(SearchSpecification specsSet, SearchSet searchSet) {
var limits = specsSet.queryLimits;
this.fetchSize = limits.fetchSize();
this.budget = new IndexSearchBudget(limits.timeoutMs());
this.fetchSize = limits.getFetchSize();
this.budget = new IndexSearchBudget(limits.getTimeoutMs());
this.query = specsSet.query;
this.limitByDomain = limits.resultsByDomain();
this.limitTotal = limits.resultsTotal();
this.limitByDomain = limits.getResultsByDomain();
this.limitTotal = limits.getResultsTotal();
queryParams = new QueryParams(
specsSet.quality,
@ -62,17 +65,17 @@ public class SearchParameters {
}
public SearchParameters(RpcIndexQuery request, SearchSet searchSet) {
var limits = IndexProtobufCodec.convertQueryLimits(request.getQueryLimits());
var limits = request.getQueryLimits();
this.fetchSize = limits.fetchSize();
this.fetchSize = limits.getFetchSize();
// The time budget is halved because this is the point when we start to
// wrap up the search and return the results.
this.budget = new IndexSearchBudget(limits.timeoutMs() / 2);
this.budget = new IndexSearchBudget(limits.getTimeoutMs() / 2);
this.query = IndexProtobufCodec.convertRpcQuery(request.getQuery());
this.limitByDomain = limits.resultsByDomain();
this.limitTotal = limits.resultsTotal();
this.limitByDomain = limits.getResultsByDomain();
this.limitTotal = limits.getResultsTotal();
queryParams = new QueryParams(
convertSpecLimit(request.getQuality()),
@ -85,7 +88,7 @@ public class SearchParameters {
compiledQuery = CompiledQueryParser.parse(this.query.compiledQuery);
compiledQueryIds = compiledQuery.mapToLong(SearchTermsUtil::getWordId);
rankingParams = IndexProtobufCodec.convertRankingParameterss(request.getParameters());
rankingParams = Objects.requireNonNullElseGet(request.getParameters(), PrototypeRankingParameters::sensibleDefaults);
}

7
code/index/java/nu/marginalia/index/results/Bm25GraphVisitor.java
View File

@ -2,7 +2,6 @@ package nu.marginalia.index.results;
import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
import nu.marginalia.api.searchquery.model.compiled.CqExpression;
import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
import java.util.BitSet;
@ -24,14 +23,14 @@ public class Bm25GraphVisitor implements CqExpression.DoubleVisitor {
private final BitSet mask;
public Bm25GraphVisitor(Bm25Parameters bm25Parameters,
public Bm25GraphVisitor(double k1, double b,
float[] counts,
int length,
ResultRankingContext ctx) {
this.length = length;
this.k1 = bm25Parameters.k();
this.b = bm25Parameters.b();
this.k1 = k1;
this.b = b;
this.docCount = ctx.termFreqDocCount();
this.counts = counts;

2
code/index/java/nu/marginalia/index/results/IndexResultRankingService.java
View File

@ -156,7 +156,7 @@ public class IndexResultRankingService {
// for the selected results, as this would be comically expensive to do for all the results we
// discard along the way
if (params.rankingParams.exportDebugData) {
if (params.rankingParams.getExportDebugData()) {
var combinedIdsList = new LongArrayList(resultsList.size());
for (var item : resultsList) {
combinedIdsList.add(item.combinedId);

41
code/index/java/nu/marginalia/index/results/IndexResultScoreCalculator.java
View File

@ -2,10 +2,11 @@ package nu.marginalia.index.results;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntList;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.api.searchquery.model.compiled.CompiledQuery;
import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong;
import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.SearchResultItem;
import nu.marginalia.api.searchquery.model.results.debug.DebugRankingFactors;
import nu.marginalia.index.forward.spans.DocumentSpans;
@ -116,14 +117,14 @@ public class IndexResultScoreCalculator {
float proximitiyFac = getProximitiyFac(decodedPositions, searchTerms.phraseConstraints, verbatimMatches, unorderedMatches, spans);
double score_firstPosition = params.tcfFirstPosition * (1.0 / Math.sqrt(unorderedMatches.firstPosition));
double score_verbatim = params.tcfVerbatim * verbatimMatches.getScore();
double score_proximity = params.tcfProximity * proximitiyFac;
double score_bM25 = params.bm25Weight
* wordFlagsQuery.root.visit(new Bm25GraphVisitor(params.bm25Params, unorderedMatches.getWeightedCounts(), docSize, rankingContext))
double score_firstPosition = params.getTcfFirstPositionWeight() * (1.0 / Math.sqrt(unorderedMatches.firstPosition));
double score_verbatim = params.getTcfVerbatimWeight() * verbatimMatches.getScore();
double score_proximity = params.getTcfProximityWeight() * proximitiyFac;
double score_bM25 = params.getBm25Weight()
* wordFlagsQuery.root.visit(new Bm25GraphVisitor(params.getBm25K(), params.getBm25B(), unorderedMatches.getWeightedCounts(), docSize, rankingContext))
/ (Math.sqrt(unorderedMatches.searchableKeywordCount + 1));
double score_bFlags = params.bm25Weight
* wordFlagsQuery.root.visit(new TermFlagsGraphVisitor(params.bm25Params, wordFlagsQuery.data, unorderedMatches.getWeightedCounts(), rankingContext))
double score_bFlags = params.getBm25Weight()
* wordFlagsQuery.root.visit(new TermFlagsGraphVisitor(params.getBm25K(), wordFlagsQuery.data, unorderedMatches.getWeightedCounts(), rankingContext))
/ (Math.sqrt(unorderedMatches.searchableKeywordCount + 1));
double score = normalize(
@ -245,10 +246,10 @@ public class IndexResultScoreCalculator {
private double calculateDocumentBonus(long documentMetadata,
int features,
int length,
ResultRankingParameters rankingParams,
RpcResultRankingParameters rankingParams,
@Nullable DebugRankingFactors debugRankingFactors) {
if (rankingParams.disablePenalties) {
if (rankingParams.getDisablePenalties()) {
return 0.;
}
@ -260,18 +261,18 @@ public class IndexResultScoreCalculator {
int topology = DocumentMetadata.decodeTopology(documentMetadata);
int year = DocumentMetadata.decodeYear(documentMetadata);
double averageSentenceLengthPenalty = (asl >= rankingParams.shortSentenceThreshold ? 0 : -rankingParams.shortSentencePenalty);
double averageSentenceLengthPenalty = (asl >= rankingParams.getShortSentenceThreshold() ? 0 : -rankingParams.getShortSentencePenalty());
final double qualityPenalty = calculateQualityPenalty(size, quality, rankingParams);
final double rankingBonus = (255. - rank) * rankingParams.domainRankBonus;
final double rankingBonus = (255. - rank) * rankingParams.getDomainRankBonus();
final double topologyBonus = Math.log(1 + topology);
final double documentLengthPenalty = length > rankingParams.shortDocumentThreshold ? 0 : -rankingParams.shortDocumentPenalty;
final double documentLengthPenalty = length > rankingParams.getShortDocumentThreshold() ? 0 : -rankingParams.getShortDocumentPenalty();
final double temporalBias;
if (rankingParams.temporalBias == ResultRankingParameters.TemporalBias.RECENT) {
temporalBias = - Math.abs(year - PubDate.MAX_YEAR) * rankingParams.temporalBiasWeight;
} else if (rankingParams.temporalBias == ResultRankingParameters.TemporalBias.OLD) {
temporalBias = - Math.abs(year - PubDate.MIN_YEAR) * rankingParams.temporalBiasWeight;
if (rankingParams.getTemporalBias().getBias() == RpcTemporalBias.Bias.RECENT) {
temporalBias = - Math.abs(year - PubDate.MAX_YEAR) * rankingParams.getTemporalBiasWeight();
} else if (rankingParams.getTemporalBias().getBias() == RpcTemporalBias.Bias.OLD) {
temporalBias = - Math.abs(year - PubDate.MIN_YEAR) * rankingParams.getTemporalBiasWeight();
} else {
temporalBias = 0;
}
@ -510,14 +511,14 @@ public class IndexResultScoreCalculator {
}
private double calculateQualityPenalty(int size, int quality, ResultRankingParameters rankingParams) {
private double calculateQualityPenalty(int size, int quality, RpcResultRankingParameters rankingParams) {
if (size < 400) {
if (quality < 5)
return 0;
return -quality * rankingParams.qualityPenalty;
return -quality * rankingParams.getQualityPenalty();
}
else {
return -quality * rankingParams.qualityPenalty * 20;
return -quality * rankingParams.getQualityPenalty() * 20;
}
}

10
code/index/java/nu/marginalia/index/results/TermFlagsGraphVisitor.java
View File

@ -3,7 +3,6 @@ package nu.marginalia.index.results;
import nu.marginalia.api.searchquery.model.compiled.CqDataInt;
import nu.marginalia.api.searchquery.model.compiled.CqDataLong;
import nu.marginalia.api.searchquery.model.compiled.CqExpression;
import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
import nu.marginalia.api.searchquery.model.results.ResultRankingContext;
import nu.marginalia.model.idx.WordFlags;
@ -15,15 +14,14 @@ public class TermFlagsGraphVisitor implements CqExpression.DoubleVisitor {
private final CqDataLong wordMetaData;
private final CqDataInt frequencies;
private final float[] counts;
private final Bm25Parameters bm25Parameters;
private final double k1;
private final int docCount;
public TermFlagsGraphVisitor(Bm25Parameters bm25Parameters,
public TermFlagsGraphVisitor(double k1,
CqDataLong wordMetaData,
float[] counts,
ResultRankingContext ctx) {
this.bm25Parameters = bm25Parameters;
this.k1 = k1;
this.counts = counts;
this.docCount = ctx.termFreqDocCount();
this.wordMetaData = wordMetaData;
@ -55,7 +53,7 @@ public class TermFlagsGraphVisitor implements CqExpression.DoubleVisitor {
int freq = frequencies.get(idx);
// note we override b to zero for priority terms as they are independent of document length
return invFreq(docCount, freq) * f(bm25Parameters.k(), 0, count, 0);
return invFreq(docCount, freq) * f(k1, 0, count, 0);
}
private double evaluatePriorityScore(int idx) {

7
code/index/query/java/nu/marginalia/index/query/limit/QueryLimits.java
View File

@ -1,7 +0,0 @@
package nu.marginalia.index.query.limit;
public record QueryLimits(int resultsByDomain, int resultsTotal, int timeoutMs, int fetchSize) {
public QueryLimits forSingleDomain() {
return new QueryLimits(resultsTotal, resultsTotal, timeoutMs, fetchSize);
}
}

48
code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java
View File

@ -4,10 +4,11 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import nu.marginalia.IndexLocations;
import nu.marginalia.api.searchquery.RpcDecoratedResultItem;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.query.SearchSpecification;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.index.construction.DocIdRewriter;
import nu.marginalia.index.construction.full.FullIndexConstructor;
import nu.marginalia.index.construction.prio.PrioIndexConstructor;
@ -17,7 +18,6 @@ import nu.marginalia.index.forward.construction.ForwardIndexConverter;
import nu.marginalia.index.index.StatefulIndex;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.journal.IndexJournalSlopWriter;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.linkdb.docs.DocumentDbReader;
@ -115,9 +115,16 @@ public class IndexQueryServiceIntegrationSmokeTest {
var rsp = queryService.justQuery(
SearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.queryLimits(
RpcQueryLimits.newBuilder()
.setResultsByDomain(10)
.setResultsTotal(10)
.setTimeoutMs(Integer.MAX_VALUE)
.setFetchSize(4000)
.build()
)
.queryStrategy(QueryStrategy.SENTENCE)
.rankingParams(ResultRankingParameters.sensibleDefaults())
.rankingParams(PrototypeRankingParameters.sensibleDefaults())
.domains(new ArrayList<>())
.searchSetIdentifier("NONE")
.query(
@ -171,9 +178,16 @@ public class IndexQueryServiceIntegrationSmokeTest {
var rsp = queryService.justQuery(
SearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.queryLimits(
RpcQueryLimits.newBuilder()
.setResultsByDomain(10)
.setResultsTotal(10)
.setTimeoutMs(Integer.MAX_VALUE)
.setFetchSize(4000)
.build()
)
.queryStrategy(QueryStrategy.SENTENCE)
.rankingParams(ResultRankingParameters.sensibleDefaults())
.rankingParams(PrototypeRankingParameters.sensibleDefaults())
.domains(new ArrayList<>())
.searchSetIdentifier("NONE")
.query(
@ -225,8 +239,15 @@ public class IndexQueryServiceIntegrationSmokeTest {
var rsp = queryService.justQuery(
SearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.rankingParams(ResultRankingParameters.sensibleDefaults())
.queryLimits(
RpcQueryLimits.newBuilder()
.setResultsByDomain(10)
.setResultsTotal(10)
.setTimeoutMs(Integer.MAX_VALUE)
.setFetchSize(4000)
.build()
)
.rankingParams(PrototypeRankingParameters.sensibleDefaults())
.queryStrategy(QueryStrategy.SENTENCE)
.domains(List.of(2))
.query(
@ -282,11 +303,18 @@ public class IndexQueryServiceIntegrationSmokeTest {
var rsp = queryService.justQuery(
SearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.queryLimits(
RpcQueryLimits.newBuilder()
.setResultsByDomain(10)
.setResultsTotal(10)
.setTimeoutMs(Integer.MAX_VALUE)
.setFetchSize(4000)
.build()
)
.year(SpecificationLimit.equals(1998))
.queryStrategy(QueryStrategy.SENTENCE)
.searchSetIdentifier("NONE")
.rankingParams(ResultRankingParameters.sensibleDefaults())
.rankingParams(PrototypeRankingParameters.sensibleDefaults())
.query(
SearchQuery.builder()
.compiledQuery("4")

15
code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java
View File

@ -4,10 +4,11 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import it.unimi.dsi.fastutil.ints.IntList;
import nu.marginalia.IndexLocations;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.query.SearchSpecification;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.hash.MurmurHash3_128;
import nu.marginalia.index.construction.DocIdRewriter;
import nu.marginalia.index.construction.full.FullIndexConstructor;
@ -18,7 +19,6 @@ import nu.marginalia.index.forward.construction.ForwardIndexConverter;
import nu.marginalia.index.index.StatefulIndex;
import nu.marginalia.index.journal.IndexJournal;
import nu.marginalia.index.journal.IndexJournalSlopWriter;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.linkdb.docs.DocumentDbReader;
@ -389,13 +389,20 @@ public class IndexQueryServiceIntegrationTest {
SearchSpecification basicQuery(Function<SearchSpecification.SearchSpecificationBuilder, SearchSpecification.SearchSpecificationBuilder> mutator)
{
var builder = SearchSpecification.builder()
.queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000))
.queryLimits(
RpcQueryLimits.newBuilder()
.setResultsByDomain(10)
.setResultsTotal(10)
.setTimeoutMs(Integer.MAX_VALUE)
.setFetchSize(4000)
.build()
)
.queryStrategy(QueryStrategy.SENTENCE)
.year(SpecificationLimit.none())
.quality(SpecificationLimit.none())
.size(SpecificationLimit.none())
.rank(SpecificationLimit.none())
.rankingParams(ResultRankingParameters.sensibleDefaults())
.rankingParams(PrototypeRankingParameters.sensibleDefaults())
.domains(new ArrayList<>())
.searchSetIdentifier("NONE");

1
code/services-application/api-service/build.gradle
View File

@ -44,6 +44,7 @@ dependencies {
implementation libs.bundles.jetty
implementation libs.opencsv
implementation libs.trove
implementation libs.protobuf
implementation libs.fastutil
implementation libs.bundles.gson
implementation libs.bundles.mariadb

13
code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java
View File

@ -6,10 +6,10 @@ import nu.marginalia.api.model.ApiSearchResult;
import nu.marginalia.api.model.ApiSearchResultQueryDetails;
import nu.marginalia.api.model.ApiSearchResults;
import nu.marginalia.api.searchquery.QueryClient;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier;
import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.model.idx.WordFlags;
import java.util.ArrayList;
@ -47,11 +47,12 @@ public class ApiSearchOperator {
return new QueryParams(
query,
new QueryLimits(
2,
Math.min(100, count),
150,
8192),
RpcQueryLimits.newBuilder()
.setResultsByDomain(2)
.setResultsTotal(Math.min(100, count))
.setTimeoutMs(150)
.setFetchSize(8192)
.build(),
searchSet.name());
}

8
code/services-application/search-service-legacy/java/nu/marginalia/search/SearchOperator.java
View File

@ -5,11 +5,11 @@ import com.google.inject.Singleton;
import nu.marginalia.WebsiteUrl;
import nu.marginalia.api.math.MathClient;
import nu.marginalia.api.searchquery.QueryClient;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.QueryResponse;
import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
import nu.marginalia.bbpc.BrailleBlockPunchCards;
import nu.marginalia.db.DbDomainQueries;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.model.EdgeUrl;
import nu.marginalia.model.crawl.DomainIndexingState;
@ -155,15 +155,15 @@ public class SearchOperator {
public List<UrlDetails> getResultsFromQuery(QueryResponse queryResponse) {
final QueryLimits limits = queryResponse.specs().queryLimits;
final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.resultsByDomain());
final RpcQueryLimits limits = queryResponse.specs().queryLimits;
final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.getResultsByDomain());
// Update the query count (this is what you see on the front page)
searchVisitorCount.registerQuery();
return queryResponse.results().stream()
.filter(deduplicator::shouldRetain)
.limit(limits.resultsTotal())
.limit(limits.getResultsTotal())
.map(SearchOperator::createDetails)
.toList();
}

38
code/services-application/search-service-legacy/java/nu/marginalia/search/SearchQueryParamFactory.java
View File

@ -1,10 +1,10 @@
package nu.marginalia.search;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.search.command.SearchParameters;
@ -12,6 +12,21 @@ import nu.marginalia.search.command.SearchParameters;
import java.util.List;
public class SearchQueryParamFactory {
static final RpcQueryLimits defaultLimits = RpcQueryLimits.newBuilder()
.setResultsTotal(100)
.setResultsByDomain(5)
.setTimeoutMs(200)
.setFetchSize(8192)
.build();
static final RpcQueryLimits shallowLimit = RpcQueryLimits.newBuilder()
.setResultsTotal(100)
.setResultsByDomain(100)
.setTimeoutMs(100)
.setFetchSize(512)
.build();
public QueryParams forRegularSearch(SearchParameters userParams) {
SearchQuery prototype = new SearchQuery();
@ -33,7 +48,7 @@ public class SearchQueryParamFactory {
profile.getSizeLimit(),
SpecificationLimit.none(),
List.of(),
new QueryLimits(5, 100, 200, 8192),
defaultLimits,
profile.searchSetIdentifier.name(),
userParams.strategy(),
userParams.temporalBias(),
@ -54,10 +69,15 @@ public class SearchQueryParamFactory {
SpecificationLimit.none(),
SpecificationLimit.none(),
List.of(domainId),
new QueryLimits(count, count, 100, 512),
RpcQueryLimits.newBuilder()
.setResultsTotal(count)
.setResultsByDomain(count)
.setTimeoutMs(100)
.setFetchSize(512)
.build(),
SearchSetIdentifier.NONE.name(),
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
1
);
}
@ -74,10 +94,10 @@ public class SearchQueryParamFactory {
SpecificationLimit.none(),
SpecificationLimit.none(),
List.of(),
new QueryLimits(100, 100, 100, 512),
shallowLimit,
SearchSetIdentifier.NONE.name(),
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
1
);
}
@ -94,10 +114,10 @@ public class SearchQueryParamFactory {
SpecificationLimit.none(),
SpecificationLimit.none(),
List.of(),
new QueryLimits(100, 100, 100, 512),
shallowLimit,
SearchSetIdentifier.NONE.name(),
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
1
);
}

10
code/services-application/search-service-legacy/java/nu/marginalia/search/command/SearchParameters.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.search.command;
import nu.marginalia.WebsiteUrl;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.search.model.SearchProfile;
@ -78,15 +78,15 @@ public record SearchParameters(String query,
return baseUrl.withPath(path);
}
public ResultRankingParameters.TemporalBias temporalBias() {
public RpcTemporalBias.Bias temporalBias() {
if (recent == RECENT) {
return ResultRankingParameters.TemporalBias.RECENT;
return RpcTemporalBias.Bias.RECENT;
}
else if (profile == SearchProfile.VINTAGE) {
return ResultRankingParameters.TemporalBias.OLD;
return RpcTemporalBias.Bias.OLD;
}
return ResultRankingParameters.TemporalBias.NONE;
return RpcTemporalBias.Bias.NONE;
}
public QueryStrategy strategy() {

12
code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java
View File

@ -2,14 +2,13 @@ package nu.marginalia.search;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import nu.marginalia.WebsiteUrl;
import nu.marginalia.api.math.MathClient;
import nu.marginalia.api.searchquery.QueryClient;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.QueryResponse;
import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem;
import nu.marginalia.bbpc.BrailleBlockPunchCards;
import nu.marginalia.db.DbDomainQueries;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.model.EdgeUrl;
import nu.marginalia.model.crawl.DomainIndexingState;
@ -47,7 +46,6 @@ public class SearchOperator {
private final DbDomainQueries domainQueries;
private final QueryClient queryClient;
private final SearchQueryParamFactory paramFactory;
private final WebsiteUrl websiteUrl;
private final SearchUnitConversionService searchUnitConversionService;
private final SearchQueryCountService searchVisitorCount;
@ -57,7 +55,6 @@ public class SearchOperator {
DbDomainQueries domainQueries,
QueryClient queryClient,
SearchQueryParamFactory paramFactory,
WebsiteUrl websiteUrl,
SearchUnitConversionService searchUnitConversionService,
SearchQueryCountService searchVisitorCount
)
@ -67,7 +64,6 @@ public class SearchOperator {
this.domainQueries = domainQueries;
this.queryClient = queryClient;
this.paramFactory = paramFactory;
this.websiteUrl = websiteUrl;
this.searchUnitConversionService = searchUnitConversionService;
this.searchVisitorCount = searchVisitorCount;
}
@ -154,8 +150,8 @@ public class SearchOperator {
public SimpleSearchResults getResultsFromQuery(QueryResponse queryResponse) {
final QueryLimits limits = queryResponse.specs().queryLimits;
final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.resultsByDomain());
final RpcQueryLimits limits = queryResponse.specs().queryLimits;
final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.getResultsByDomain());
// Update the query count (this is what you see on the front page)
searchVisitorCount.registerQuery();
@ -164,7 +160,7 @@ public class SearchOperator {
.sorted(this::retentionSortOrder) // Sort in an order that makes us more likely to discard the "bad" duplicates
.filter(deduplicator::shouldRetain)
.sorted() // Return to the presentation sort order before limiting so we don't throw out good results over schema and "ip-ness"
.limit(limits.resultsTotal())
.limit(limits.getResultsTotal())
.map(SearchOperator::createDetails)
.toList();

41
code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java
View File

@ -1,10 +1,10 @@
package nu.marginalia.search;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.search.command.SearchParameters;
@ -13,6 +13,22 @@ import java.util.List;
public class SearchQueryParamFactory {
static final RpcQueryLimits defaultLimits = RpcQueryLimits.newBuilder()
.setResultsTotal(100)
.setResultsByDomain(5)
.setTimeoutMs(200)
.setFetchSize(8192)
.build();
static final RpcQueryLimits shallowLimit = RpcQueryLimits.newBuilder()
.setResultsTotal(100)
.setResultsByDomain(100)
.setTimeoutMs(100)
.setFetchSize(512)
.build();
public QueryParams forRegularSearch(SearchParameters userParams) {
SearchQuery prototype = new SearchQuery();
var profile = userParams.profile();
@ -29,11 +45,11 @@ public class SearchQueryParamFactory {
prototype.searchTermsPriority,
prototype.searchTermsAdvice,
profile.getQualityLimit(),
profile.getYearLimit(),
userParams.yearLimit(),
profile.getSizeLimit(),
SpecificationLimit.none(),
List.of(),
new QueryLimits(5, 100, 200, 8192),
defaultLimits,
profile.searchSetIdentifier.name(),
userParams.strategy(),
userParams.temporalBias(),
@ -54,10 +70,15 @@ public class SearchQueryParamFactory {
SpecificationLimit.none(),
SpecificationLimit.none(),
List.of(domainId),
new QueryLimits(count, count, 100, 512),
RpcQueryLimits.newBuilder()
.setResultsTotal(count)
.setResultsByDomain(count)
.setTimeoutMs(100)
.setFetchSize(512)
.build(),
SearchSetIdentifier.NONE.name(),
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
page
);
}
@ -74,10 +95,10 @@ public class SearchQueryParamFactory {
SpecificationLimit.none(),
SpecificationLimit.none(),
List.of(),
new QueryLimits(100, 100, 100, 512),
shallowLimit,
SearchSetIdentifier.NONE.name(),
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
page
);
}
@ -94,10 +115,10 @@ public class SearchQueryParamFactory {
SpecificationLimit.none(),
SpecificationLimit.none(),
List.of(),
new QueryLimits(100, 100, 100, 512),
shallowLimit,
SearchSetIdentifier.NONE.name(),
QueryStrategy.AUTO,
ResultRankingParameters.TemporalBias.NONE,
RpcTemporalBias.Bias.NONE,
1
);
}

10
code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java
View File

@ -1,7 +1,7 @@
package nu.marginalia.search.command;
import nu.marginalia.WebsiteUrl;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.index.query.limit.QueryStrategy;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.model.EdgeDomain;
@ -98,15 +98,15 @@ public record SearchParameters(WebsiteUrl url,
return path;
}
public ResultRankingParameters.TemporalBias temporalBias() {
public RpcTemporalBias.Bias temporalBias() {
if (recent == RECENT) {
return ResultRankingParameters.TemporalBias.RECENT;
return RpcTemporalBias.Bias.RECENT;
}
else if (profile == SearchProfile.VINTAGE) {
return ResultRankingParameters.TemporalBias.OLD;
return RpcTemporalBias.Bias.OLD;
}
return ResultRankingParameters.TemporalBias.NONE;
return RpcTemporalBias.Bias.NONE;
}
public QueryStrategy strategy() {

1
code/services-core/control-service/build.gradle
View File

@ -55,6 +55,7 @@ dependencies {
implementation libs.duckdb
implementation libs.jsoup
implementation libs.protobuf
implementation libs.trove
implementation dependencies.create(libs.spark.get()) {

16
code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java
View File

@ -2,11 +2,10 @@ package nu.marginalia.control.app.svc;
import com.google.inject.Inject;
import nu.marginalia.api.searchquery.QueryClient;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.control.ControlRendererFactory;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.model.EdgeUrl;
import nu.marginalia.nodecfg.NodeConfigurationService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.Request;
@ -22,18 +21,16 @@ public class SearchToBanService {
private final ControlRendererFactory rendererFactory;
private final QueryClient queryClient;
private final Logger logger = LoggerFactory.getLogger(getClass());
private final NodeConfigurationService nodeConfigurationService;
@Inject
public SearchToBanService(ControlBlacklistService blacklistService,
ControlRendererFactory rendererFactory,
QueryClient queryClient, NodeConfigurationService nodeConfigurationService)
QueryClient queryClient)
{
this.blacklistService = blacklistService;
this.rendererFactory = rendererFactory;
this.queryClient = queryClient;
this.nodeConfigurationService = nodeConfigurationService;
}
public void register() throws IOException {
@ -76,7 +73,14 @@ public class SearchToBanService {
private Object executeQuery(String query) {
return queryClient.search(new QueryParams(
query, new QueryLimits(2, 200, 250, 8192),
query,
RpcQueryLimits.newBuilder()
.setResultsTotal(100)
.setResultsByDomain(2)
.setTimeoutMs(200)
.setFetchSize(8192)
.build()
,
"NONE"
));
}

75
code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java
View File

@ -3,12 +3,13 @@ package nu.marginalia.query;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.inject.Inject;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.RpcTemporalBias;
import nu.marginalia.api.searchquery.model.query.QueryParams;
import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.functions.searchquery.QueryGRPCService;
import nu.marginalia.index.api.IndexClient;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.renderer.MustacheRenderer;
import nu.marginalia.renderer.RendererFactory;
@ -53,9 +54,14 @@ public class QueryBasicInterface {
int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5"));
String set = requireNonNullElse(request.queryParams("set"), "");
var params = new QueryParams(queryString, new QueryLimits(
domainCount, min(100, count * 10), 250, 8192
), set);
var params = new QueryParams(queryString,
RpcQueryLimits.newBuilder()
.setResultsByDomain(domainCount)
.setResultsTotal(min(100, count * 10))
.setTimeoutMs(250)
.setFetchSize(8192)
.build()
, set);
var pagination = new IndexClient.Pagination(page, count);
@ -63,7 +69,7 @@ public class QueryBasicInterface {
queryString,
params,
pagination,
ResultRankingParameters.sensibleDefaults()
PrototypeRankingParameters.sensibleDefaults()
);
var results = detailedDirectResult.result();
@ -92,7 +98,7 @@ public class QueryBasicInterface {
String queryString = request.queryParams("q");
if (queryString == null) {
// Show the default query form if no query is given
return qdebugRenderer.render(Map.of("rankingParams", ResultRankingParameters.sensibleDefaults())
return qdebugRenderer.render(Map.of("rankingParams", PrototypeRankingParameters.sensibleDefaults())
);
}
@ -101,9 +107,14 @@ public class QueryBasicInterface {
int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5"));
String set = requireNonNullElse(request.queryParams("set"), "");
var queryParams = new QueryParams(queryString, new QueryLimits(
domainCount, min(100, count * 10), 250, 8192
), set);
var queryParams = new QueryParams(queryString,
RpcQueryLimits.newBuilder()
.setResultsByDomain(domainCount)
.setResultsTotal(min(100, count * 10))
.setTimeoutMs(250)
.setFetchSize(8192)
.build(),
set);
var pagination = new IndexClient.Pagination(page, count);
@ -126,28 +137,28 @@ public class QueryBasicInterface {
);
}
private ResultRankingParameters debugRankingParamsFromRequest(Request request) {
var sensibleDefaults = ResultRankingParameters.sensibleDefaults();
private RpcResultRankingParameters debugRankingParamsFromRequest(Request request) {
var sensibleDefaults = PrototypeRankingParameters.sensibleDefaults();
return ResultRankingParameters.builder()
.domainRankBonus(doubleFromRequest(request, "domainRankBonus", sensibleDefaults.domainRankBonus))
.qualityPenalty(doubleFromRequest(request, "qualityPenalty", sensibleDefaults.qualityPenalty))
.shortDocumentThreshold(intFromRequest(request, "shortDocumentThreshold", sensibleDefaults.shortDocumentThreshold))
.shortDocumentPenalty(doubleFromRequest(request, "shortDocumentPenalty", sensibleDefaults.shortDocumentPenalty))
.tcfFirstPosition(doubleFromRequest(request, "tcfFirstPosition", sensibleDefaults.tcfFirstPosition))
.tcfVerbatim(doubleFromRequest(request, "tcfVerbatim", sensibleDefaults.tcfVerbatim))
.tcfProximity(doubleFromRequest(request, "tcfProximity", sensibleDefaults.tcfProximity))
.bm25Params(new Bm25Parameters(
doubleFromRequest(request, "bm25.k1", sensibleDefaults.bm25Params.k()),
doubleFromRequest(request, "bm25.b", sensibleDefaults.bm25Params.b())
))
.temporalBias(ResultRankingParameters.TemporalBias.valueOf(stringFromRequest(request, "temporalBias", sensibleDefaults.temporalBias.toString())))
.temporalBiasWeight(doubleFromRequest(request, "temporalBiasWeight", sensibleDefaults.temporalBiasWeight))
.shortSentenceThreshold(intFromRequest(request, "shortSentenceThreshold", sensibleDefaults.shortSentenceThreshold))
.shortSentencePenalty(doubleFromRequest(request, "shortSentencePenalty", sensibleDefaults.shortSentencePenalty))
.bm25Weight(doubleFromRequest(request, "bm25Weight", sensibleDefaults.bm25Weight))
.disablePenalties(boolFromRequest(request, "disablePenalties", sensibleDefaults.disablePenalties))
.exportDebugData(true)
var bias = RpcTemporalBias.Bias.valueOf(stringFromRequest(request, "temporalBias", "NONE"));
return RpcResultRankingParameters.newBuilder()
.setDomainRankBonus(doubleFromRequest(request, "domainRankBonus", sensibleDefaults.getDomainRankBonus()))
.setQualityPenalty(doubleFromRequest(request, "qualityPenalty", sensibleDefaults.getQualityPenalty()))
.setShortDocumentThreshold(intFromRequest(request, "shortDocumentThreshold", sensibleDefaults.getShortDocumentThreshold()))
.setShortDocumentPenalty(doubleFromRequest(request, "shortDocumentPenalty", sensibleDefaults.getShortDocumentPenalty()))
.setTcfFirstPositionWeight(doubleFromRequest(request, "tcfFirstPositionWeight", sensibleDefaults.getTcfFirstPositionWeight()))
.setTcfVerbatimWeight(doubleFromRequest(request, "tcfVerbatimWeight", sensibleDefaults.getTcfVerbatimWeight()))
.setTcfProximityWeight(doubleFromRequest(request, "tcfProximityWeight", sensibleDefaults.getTcfProximityWeight()))
.setBm25B(doubleFromRequest(request, "bm25b", sensibleDefaults.getBm25B()))
.setBm25K(doubleFromRequest(request, "bm25k", sensibleDefaults.getBm25K()))
.setTemporalBias(RpcTemporalBias.newBuilder().setBias(bias).build())
.setTemporalBiasWeight(doubleFromRequest(request, "temporalBiasWeight", sensibleDefaults.getTemporalBiasWeight()))
.setShortSentenceThreshold(intFromRequest(request, "shortSentenceThreshold", sensibleDefaults.getShortSentenceThreshold()))
.setShortSentencePenalty(doubleFromRequest(request, "shortSentencePenalty", sensibleDefaults.getShortSentencePenalty()))
.setBm25Weight(doubleFromRequest(request, "bm25Weight", sensibleDefaults.getBm25Weight()))
.setDisablePenalties(boolFromRequest(request, "disablePenalties", sensibleDefaults.getDisablePenalties()))
.setExportDebugData(true)
.build();
}

20
code/services-core/query-service/resources/templates/qdebug.hdb
View File

@ -31,20 +31,20 @@
<div class="col-sm-2"><input type="text" class="form-control" id="shortDocumentPenalty" name="shortDocumentPenalty" value="{{shortDocumentPenalty}}"></div>
</div>
<div class="row my-2">
<div class="col-sm-2"><label for="tcfFirstPosition">TCF First Position Weight</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="tcfFirstPosition" name="tcfFirstPosition" value="{{tcfFirstPosition}}"></div>
<div class="col-sm-2"><label for="tcfFirstPositionWeight">TCF First Position Weight</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="tcfFirstPositionWeight" name="tcfFirstPositionWeight" value="{{tcfFirstPositionWeight}}"></div>
</div>
<div class="row my-2">
<div class="col-sm-2"><label for="tcfVerbatim">TCF Verbatim</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="tcfVerbatim" name="tcfVerbatim" value="{{tcfVerbatim}}"></div>
<div class="col-sm-2"><label for="tcfProximity">TCF Proximity</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="tcfProximity" name="tcfProximity" value="{{tcfProximity}}"></div>
<div class="col-sm-2"><label for="tcfVerbatimWeight">TCF Verbatim</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="tcfVerbatimWeight" name="tcfVerbatimWeight" value="{{tcfVerbatimWeight}}"></div>
<div class="col-sm-2"><label for="tcfProximityWeight">TCF Proximity</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="tcfProximityWeight" name="tcfProximityWeight" value="{{tcfProximityWeight}}"></div>
</div>
<div class="row my-2">
<div class="col-sm-2"><label for="bm25.k1">BM25 K1</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="bm25.k1" name="bm25.k1" value="{{bm25Params.k}}"></div>
<div class="col-sm-2"><label for="bm25.b">BM25 B</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="bm25.b" name="bm25.b" value="{{bm25Params.b}}"></div>
<div class="col-sm-2"><label for="bm25k">BM25 K1</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="bm25k" name="bm25k" value="{{bm25K}}"></div>
<div class="col-sm-2"><label for="bm25b">BM25 B</label></div>
<div class="col-sm-2"><input type="text" class="form-control" id="bm25b" name="bm25b" value="{{bm25B}}"></div>
</div>
<div class="row my-2">
<div class="col-sm-2"><label for="temporalBias">Temporal Bias</label></div>

6
code/tools/integration-test/test/nu/marginalia/IntegrationTest.java
View File

@ -5,7 +5,8 @@ import com.google.inject.Inject;
import nu.marginalia.api.searchquery.QueryProtobufCodec;
import nu.marginalia.api.searchquery.RpcQsQuery;
import nu.marginalia.api.searchquery.RpcQueryLimits;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.RpcResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters;
import nu.marginalia.converting.processor.DomainProcessor;
import nu.marginalia.converting.writer.ConverterBatchWriter;
import nu.marginalia.crawl.fetcher.ContentTags;
@ -211,8 +212,7 @@ public class IntegrationTest {
var params = QueryProtobufCodec.convertRequest(request);
var p = ResultRankingParameters.sensibleDefaults();
p.exportDebugData = true;
var p = RpcResultRankingParameters.newBuilder(PrototypeRankingParameters.sensibleDefaults()).setExportDebugData(true).build();
var query = queryFactory.createQuery(params, p);