diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java index b0ce3aa4..6f53540d 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java @@ -2,9 +2,6 @@ package nu.marginalia.api.searchquery; import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint; import nu.marginalia.api.searchquery.model.query.SearchQuery; -import nu.marginalia.api.searchquery.model.results.Bm25Parameters; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.index.query.limit.SpecificationLimitType; @@ -27,37 +24,19 @@ public class IndexProtobufCodec { .build(); } - public static QueryLimits convertQueryLimits(RpcQueryLimits queryLimits) { - return new QueryLimits( - queryLimits.getResultsByDomain(), - queryLimits.getResultsTotal(), - queryLimits.getTimeoutMs(), - queryLimits.getFetchSize() - ); - } - - public static RpcQueryLimits convertQueryLimits(QueryLimits queryLimits) { - return RpcQueryLimits.newBuilder() - .setResultsByDomain(queryLimits.resultsByDomain()) - .setResultsTotal(queryLimits.resultsTotal()) - .setTimeoutMs(queryLimits.timeoutMs()) - .setFetchSize(queryLimits.fetchSize()) - .build(); - } - public static SearchQuery convertRpcQuery(RpcQuery query) { - List phraeConstraints = new ArrayList<>(); + List phraseConstraints = new ArrayList<>(); for (int j = 0; j < query.getPhrasesCount(); j++) { var coh = query.getPhrases(j); if (coh.getType() == RpcPhrases.TYPE.OPTIONAL) { - phraeConstraints.add(new SearchPhraseConstraint.Optional(List.copyOf(coh.getTermsList()))); + phraseConstraints.add(new SearchPhraseConstraint.Optional(List.copyOf(coh.getTermsList()))); } else if (coh.getType() == RpcPhrases.TYPE.MANDATORY) { - phraeConstraints.add(new SearchPhraseConstraint.Mandatory(List.copyOf(coh.getTermsList()))); + phraseConstraints.add(new SearchPhraseConstraint.Mandatory(List.copyOf(coh.getTermsList()))); } else if (coh.getType() == RpcPhrases.TYPE.FULL) { - phraeConstraints.add(new SearchPhraseConstraint.Full(List.copyOf(coh.getTermsList()))); + phraseConstraints.add(new SearchPhraseConstraint.Full(List.copyOf(coh.getTermsList()))); } else { throw new IllegalArgumentException("Unknown phrase constraint type: " + coh.getType()); @@ -70,7 +49,7 @@ public class IndexProtobufCodec { query.getExcludeList(), query.getAdviceList(), query.getPriorityList(), - phraeConstraints + phraseConstraints ); } @@ -103,62 +82,4 @@ public class IndexProtobufCodec { return subqueryBuilder.build(); } - public static ResultRankingParameters convertRankingParameterss(RpcResultRankingParameters params) { - if (params == null) - return ResultRankingParameters.sensibleDefaults(); - - return new ResultRankingParameters( - new Bm25Parameters(params.getBm25K(), params.getBm25B()), - params.getShortDocumentThreshold(), - params.getShortDocumentPenalty(), - params.getDomainRankBonus(), - params.getQualityPenalty(), - params.getShortSentenceThreshold(), - params.getShortSentencePenalty(), - params.getBm25Weight(), - params.getTcfFirstPositionWeight(), - params.getTcfVerbatimWeight(), - params.getTcfProximityWeight(), - ResultRankingParameters.TemporalBias.valueOf(params.getTemporalBias().getBias().name()), - params.getTemporalBiasWeight(), - params.getDisablePenalties(), - params.getExportDebugData() - ); - } - - public static RpcResultRankingParameters convertRankingParameterss(ResultRankingParameters rankingParams, - RpcTemporalBias temporalBias) - { - if (rankingParams == null) { - rankingParams = ResultRankingParameters.sensibleDefaults(); - } - - var builder = RpcResultRankingParameters.newBuilder() - .setBm25B(rankingParams.bm25Params.b()) - .setBm25K(rankingParams.bm25Params.k()) - .setShortDocumentThreshold(rankingParams.shortDocumentThreshold) - .setShortDocumentPenalty(rankingParams.shortDocumentPenalty) - .setDomainRankBonus(rankingParams.domainRankBonus) - .setQualityPenalty(rankingParams.qualityPenalty) - .setShortSentenceThreshold(rankingParams.shortSentenceThreshold) - .setShortSentencePenalty(rankingParams.shortSentencePenalty) - .setBm25Weight(rankingParams.bm25Weight) - .setTcfFirstPositionWeight(rankingParams.tcfFirstPosition) - .setTcfProximityWeight(rankingParams.tcfProximity) - .setTcfVerbatimWeight(rankingParams.tcfVerbatim) - .setTemporalBiasWeight(rankingParams.temporalBiasWeight) - .setDisablePenalties(rankingParams.disablePenalties) - .setExportDebugData(rankingParams.exportDebugData); - - if (temporalBias != null && temporalBias.getBias() != RpcTemporalBias.Bias.NONE) { - builder.setTemporalBias(temporalBias); - } - else { - builder.setTemporalBias(RpcTemporalBias.newBuilder() - .setBias(RpcTemporalBias.Bias.valueOf(rankingParams.temporalBias.name()))); - } - - return builder.build(); - } - } diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java index 08ed908c..3c082387 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java @@ -5,7 +5,7 @@ import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.api.searchquery.model.query.QueryResponse; import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters; import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.api.searchquery.model.results.SearchResultKeywordScore; import nu.marginalia.api.searchquery.model.results.debug.DebugFactor; @@ -15,10 +15,7 @@ import nu.marginalia.api.searchquery.model.results.debug.ResultRankingDetails; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.model.EdgeUrl; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class QueryProtobufCodec { @@ -37,7 +34,7 @@ public class QueryProtobufCodec { builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size)); builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank)); - builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits)); + builder.setQueryLimits(query.specs.queryLimits); // Query strategy may be overridden by the query, but if not, use the one from the request if (query.specs.queryStrategy != null && query.specs.queryStrategy != QueryStrategy.AUTO) @@ -45,8 +42,12 @@ public class QueryProtobufCodec { else builder.setQueryStrategy(request.getQueryStrategy()); - if (query.specs.rankingParams != null) { - builder.setParameters(IndexProtobufCodec.convertRankingParameterss(query.specs.rankingParams, request.getTemporalBias())); + if (query.specs.rankingParams != null && request.getTemporalBias().getBias() != RpcTemporalBias.Bias.NONE) { + builder.setParameters( + RpcResultRankingParameters.newBuilder(query.specs.rankingParams) + .setTemporalBias(request.getTemporalBias()) + .build() + ); } return builder.build(); @@ -65,18 +66,13 @@ public class QueryProtobufCodec { builder.setSize(IndexProtobufCodec.convertSpecLimit(query.specs.size)); builder.setRank(IndexProtobufCodec.convertSpecLimit(query.specs.rank)); - builder.setQueryLimits(IndexProtobufCodec.convertQueryLimits(query.specs.queryLimits)); + builder.setQueryLimits(query.specs.queryLimits); // Query strategy may be overridden by the query, but if not, use the one from the request builder.setQueryStrategy(query.specs.queryStrategy.name()); if (query.specs.rankingParams != null) { - builder.setParameters(IndexProtobufCodec.convertRankingParameterss( - query.specs.rankingParams, - RpcTemporalBias.newBuilder().setBias( - RpcTemporalBias.Bias.NONE) - .build()) - ); + builder.setParameters(query.specs.rankingParams); } return builder.build(); @@ -95,10 +91,10 @@ public class QueryProtobufCodec { IndexProtobufCodec.convertSpecLimit(request.getSize()), IndexProtobufCodec.convertSpecLimit(request.getRank()), request.getDomainIdsList(), - IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()), + request.getQueryLimits(), request.getSearchSetIdentifier(), QueryStrategy.valueOf(request.getQueryStrategy()), - ResultRankingParameters.TemporalBias.valueOf(request.getTemporalBias().getBias().name()), + RpcTemporalBias.Bias.valueOf(request.getTemporalBias().getBias().name()), request.getPagination().getPage() ); } @@ -294,9 +290,9 @@ public class QueryProtobufCodec { IndexProtobufCodec.convertSpecLimit(specs.getYear()), IndexProtobufCodec.convertSpecLimit(specs.getSize()), IndexProtobufCodec.convertSpecLimit(specs.getRank()), - IndexProtobufCodec.convertQueryLimits(specs.getQueryLimits()), + specs.getQueryLimits(), QueryStrategy.valueOf(specs.getQueryStrategy()), - IndexProtobufCodec.convertRankingParameterss(specs.getParameters()) + Objects.requireNonNullElseGet(specs.getParameters(), PrototypeRankingParameters::sensibleDefaults) ); } @@ -307,7 +303,7 @@ public class QueryProtobufCodec { .addAllTacitExcludes(params.tacitExcludes()) .addAllTacitPriority(params.tacitPriority()) .setHumanQuery(params.humanQuery()) - .setQueryLimits(IndexProtobufCodec.convertQueryLimits(params.limits())) + .setQueryLimits(params.limits()) .setQuality(IndexProtobufCodec.convertSpecLimit(params.quality())) .setYear(IndexProtobufCodec.convertSpecLimit(params.year())) .setSize(IndexProtobufCodec.convertSpecLimit(params.size())) @@ -319,7 +315,7 @@ public class QueryProtobufCodec { .build()) .setPagination(RpcQsQueryPagination.newBuilder() .setPage(params.page()) - .setPageSize(Math.min(100, params.limits().resultsTotal())) + .setPageSize(Math.min(100, params.limits().getResultsTotal())) .build()); if (params.nearDomain() != null) diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java index d5bf4a17..28837fe4 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/QueryParams.java @@ -1,7 +1,7 @@ package nu.marginalia.api.searchquery.model.query; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.index.query.limit.QueryLimits; +import nu.marginalia.api.searchquery.RpcQueryLimits; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; @@ -21,14 +21,14 @@ public record QueryParams( SpecificationLimit size, SpecificationLimit rank, List domainIds, - QueryLimits limits, + RpcQueryLimits limits, String identifier, QueryStrategy queryStrategy, - ResultRankingParameters.TemporalBias temporalBias, + RpcTemporalBias.Bias temporalBias, int page ) { - public QueryParams(String query, QueryLimits limits, String identifier) { + public QueryParams(String query, RpcQueryLimits limits, String identifier) { this(query, null, List.of(), List.of(), @@ -42,7 +42,7 @@ public record QueryParams( limits, identifier, QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, 1 // page ); } diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java index d4cec49a..725fc2e6 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/query/SearchSpecification.java @@ -1,7 +1,7 @@ package nu.marginalia.api.searchquery.model.query; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.index.query.limit.QueryLimits; +import nu.marginalia.api.searchquery.RpcQueryLimits; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; @@ -24,11 +24,11 @@ public class SearchSpecification { public SpecificationLimit size; public SpecificationLimit rank; - public final QueryLimits queryLimits; + public final RpcQueryLimits queryLimits; public final QueryStrategy queryStrategy; - public final ResultRankingParameters rankingParams; + public final RpcResultRankingParameters rankingParams; public SearchSpecification(SearchQuery query, List domains, @@ -38,9 +38,9 @@ public class SearchSpecification { SpecificationLimit year, SpecificationLimit size, SpecificationLimit rank, - QueryLimits queryLimits, + RpcQueryLimits queryLimits, QueryStrategy queryStrategy, - ResultRankingParameters rankingParams) + RpcResultRankingParameters rankingParams) { this.query = query; this.domains = domains; @@ -91,7 +91,7 @@ public class SearchSpecification { return this.rank; } - public QueryLimits getQueryLimits() { + public RpcQueryLimits getQueryLimits() { return this.queryLimits; } @@ -99,7 +99,7 @@ public class SearchSpecification { return this.queryStrategy; } - public ResultRankingParameters getRankingParams() { + public RpcResultRankingParameters getRankingParams() { return this.rankingParams; } @@ -120,9 +120,9 @@ public class SearchSpecification { private boolean size$set; private SpecificationLimit rank$value; private boolean rank$set; - private QueryLimits queryLimits; + private RpcQueryLimits queryLimits; private QueryStrategy queryStrategy; - private ResultRankingParameters rankingParams; + private RpcResultRankingParameters rankingParams; SearchSpecificationBuilder() { } @@ -171,7 +171,7 @@ public class SearchSpecification { return this; } - public SearchSpecificationBuilder queryLimits(QueryLimits queryLimits) { + public SearchSpecificationBuilder queryLimits(RpcQueryLimits queryLimits) { this.queryLimits = queryLimits; return this; } @@ -181,7 +181,7 @@ public class SearchSpecification { return this; } - public SearchSpecificationBuilder rankingParams(ResultRankingParameters rankingParams) { + public SearchSpecificationBuilder rankingParams(RpcResultRankingParameters rankingParams) { this.rankingParams = rankingParams; return this; } diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/PrototypeRankingParameters.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/PrototypeRankingParameters.java new file mode 100644 index 00000000..a56b783c --- /dev/null +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/PrototypeRankingParameters.java @@ -0,0 +1,33 @@ +package nu.marginalia.api.searchquery.model.results; + +import nu.marginalia.api.searchquery.RpcResultRankingParameters; +import nu.marginalia.api.searchquery.RpcTemporalBias; + +public class PrototypeRankingParameters { + + /** These are the default ranking parameters that are used when no parameters are specified. */ + + private static final RpcResultRankingParameters _sensibleDefaults = RpcResultRankingParameters.newBuilder() + .setBm25B(0.5) + .setBm25K(1.2) + .setShortDocumentThreshold(2000) + .setShortDocumentPenalty(2.) + .setDomainRankBonus(1 / 100.) + .setQualityPenalty(1 / 15.) + .setShortSentenceThreshold(2) + .setShortSentencePenalty(5) + .setBm25Weight(1.) + .setTcfVerbatimWeight(1.) + .setTcfProximityWeight(1.) + .setTcfFirstPositionWeight(5) + .setTemporalBias(RpcTemporalBias.newBuilder().setBias(RpcTemporalBias.Bias.NONE)) + .setTemporalBiasWeight(5.0) + .setExportDebugData(false) + .setDisablePenalties(false) + .build(); + + public static RpcResultRankingParameters sensibleDefaults() { + return _sensibleDefaults; + } + +} diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java index c88b5662..a0cc23e3 100644 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java +++ b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingContext.java @@ -1,12 +1,13 @@ package nu.marginalia.api.searchquery.model.results; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; import nu.marginalia.api.searchquery.model.compiled.CqDataInt; import java.util.BitSet; public class ResultRankingContext { private final int docCount; - public final ResultRankingParameters params; + public final RpcResultRankingParameters params; public final BitSet regularMask; @@ -21,7 +22,7 @@ public class ResultRankingContext { public final CqDataInt priorityCounts; public ResultRankingContext(int docCount, - ResultRankingParameters params, + RpcResultRankingParameters params, BitSet ngramsMask, BitSet regularMask, CqDataInt fullCounts, diff --git a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java b/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java deleted file mode 100644 index 01d276d9..00000000 --- a/code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/ResultRankingParameters.java +++ /dev/null @@ -1,291 +0,0 @@ -package nu.marginalia.api.searchquery.model.results; - -import java.util.Objects; - -public class ResultRankingParameters { - - /** - * Tuning for BM25 when applied to full document matches - */ - public final Bm25Parameters bm25Params; - - /** - * Documents below this length are penalized - */ - public int shortDocumentThreshold; - - public double shortDocumentPenalty; - - - /** - * Scaling factor associated with domain rank (unscaled rank value is 0-255; high is good) - */ - public double domainRankBonus; - - /** - * Scaling factor associated with document quality (unscaled rank value is 0-15; high is bad) - */ - public double qualityPenalty; - - /** - * Average sentence length values below this threshold are penalized, range [0-4), 2 or 3 is probably what you want - */ - public int shortSentenceThreshold; - - /** - * Magnitude of penalty for documents with low average sentence length - */ - public double shortSentencePenalty; - - public double bm25Weight; - public double tcfFirstPosition; - public double tcfVerbatim; - public double tcfProximity; - - - public TemporalBias temporalBias; - public double temporalBiasWeight; - - public boolean disablePenalties; - public boolean exportDebugData; - - public ResultRankingParameters(Bm25Parameters bm25Params, int shortDocumentThreshold, double shortDocumentPenalty, double domainRankBonus, double qualityPenalty, int shortSentenceThreshold, double shortSentencePenalty, double bm25Weight, double tcfFirstPosition, double tcfVerbatim, double tcfProximity, TemporalBias temporalBias, double temporalBiasWeight, boolean disablePenalties, boolean exportDebugData) { - this.bm25Params = bm25Params; - this.shortDocumentThreshold = shortDocumentThreshold; - this.shortDocumentPenalty = shortDocumentPenalty; - this.domainRankBonus = domainRankBonus; - this.qualityPenalty = qualityPenalty; - this.shortSentenceThreshold = shortSentenceThreshold; - this.shortSentencePenalty = shortSentencePenalty; - this.bm25Weight = bm25Weight; - this.tcfFirstPosition = tcfFirstPosition; - this.tcfVerbatim = tcfVerbatim; - this.tcfProximity = tcfProximity; - this.temporalBias = temporalBias; - this.temporalBiasWeight = temporalBiasWeight; - this.disablePenalties = disablePenalties; - this.exportDebugData = exportDebugData; - } - - private static final ResultRankingParameters _sensibleDefaults = builder() - .bm25Params(new Bm25Parameters(1.2, 0.5)) - .shortDocumentThreshold(2000) - .shortDocumentPenalty(2.) - .domainRankBonus(1 / 100.) - .qualityPenalty(1 / 15.) - .shortSentenceThreshold(2) - .shortSentencePenalty(5) - .bm25Weight(1.) - .tcfVerbatim(1.) - .tcfProximity(1.) - .tcfFirstPosition(5) - .temporalBias(TemporalBias.NONE) - .temporalBiasWeight(5.0) - .exportDebugData(false) - .disablePenalties(false) - .build(); - - public static ResultRankingParameters sensibleDefaults() { - return _sensibleDefaults; - } - - public static ResultRankingParametersBuilder builder() { - return new ResultRankingParametersBuilder(); - } - - public Bm25Parameters getBm25Params() { - return this.bm25Params; - } - - public int getShortDocumentThreshold() { - return this.shortDocumentThreshold; - } - - public double getShortDocumentPenalty() { - return this.shortDocumentPenalty; - } - - public double getDomainRankBonus() { - return this.domainRankBonus; - } - - public double getQualityPenalty() { - return this.qualityPenalty; - } - - public int getShortSentenceThreshold() { - return this.shortSentenceThreshold; - } - - public double getShortSentencePenalty() { - return this.shortSentencePenalty; - } - - public double getBm25Weight() { - return this.bm25Weight; - } - - public double getTcfFirstPosition() { - return this.tcfFirstPosition; - } - - public double getTcfVerbatim() { - return this.tcfVerbatim; - } - - public double getTcfProximity() { - return this.tcfProximity; - } - - public TemporalBias getTemporalBias() { - return this.temporalBias; - } - - public double getTemporalBiasWeight() { - return this.temporalBiasWeight; - } - - public boolean isDisablePenalties() { return this.disablePenalties; } - - public boolean isExportDebugData() { - return this.exportDebugData; - } - - @Override - public final boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof ResultRankingParameters that)) return false; - - return shortDocumentThreshold == that.shortDocumentThreshold && Double.compare(shortDocumentPenalty, that.shortDocumentPenalty) == 0 && Double.compare(domainRankBonus, that.domainRankBonus) == 0 && Double.compare(qualityPenalty, that.qualityPenalty) == 0 && shortSentenceThreshold == that.shortSentenceThreshold && Double.compare(shortSentencePenalty, that.shortSentencePenalty) == 0 && Double.compare(bm25Weight, that.bm25Weight) == 0 && Double.compare(tcfFirstPosition, that.tcfFirstPosition) == 0 && Double.compare(tcfVerbatim, that.tcfVerbatim) == 0 && Double.compare(tcfProximity, that.tcfProximity) == 0 && Double.compare(temporalBiasWeight, that.temporalBiasWeight) == 0 && exportDebugData == that.exportDebugData && Objects.equals(bm25Params, that.bm25Params) && temporalBias == that.temporalBias; - } - - @Override - public int hashCode() { - int result = Objects.hashCode(bm25Params); - result = 31 * result + shortDocumentThreshold; - result = 31 * result + Double.hashCode(shortDocumentPenalty); - result = 31 * result + Double.hashCode(domainRankBonus); - result = 31 * result + Double.hashCode(qualityPenalty); - result = 31 * result + shortSentenceThreshold; - result = 31 * result + Double.hashCode(shortSentencePenalty); - result = 31 * result + Double.hashCode(bm25Weight); - result = 31 * result + Double.hashCode(tcfFirstPosition); - result = 31 * result + Double.hashCode(tcfVerbatim); - result = 31 * result + Double.hashCode(tcfProximity); - result = 31 * result + Objects.hashCode(temporalBias); - result = 31 * result + Double.hashCode(temporalBiasWeight); - result = 31 * result + Boolean.hashCode(disablePenalties); - result = 31 * result + Boolean.hashCode(exportDebugData); - return result; - } - - public String toString() { - return "ResultRankingParameters(bm25Params=" + this.getBm25Params() + ", shortDocumentThreshold=" + this.getShortDocumentThreshold() + ", shortDocumentPenalty=" + this.getShortDocumentPenalty() + ", domainRankBonus=" + this.getDomainRankBonus() + ", qualityPenalty=" + this.getQualityPenalty() + ", shortSentenceThreshold=" + this.getShortSentenceThreshold() + ", shortSentencePenalty=" + this.getShortSentencePenalty() + ", bm25Weight=" + this.getBm25Weight() + ", tcfFirstPosition=" + this.getTcfFirstPosition() + ", tcfVerbatim=" + this.getTcfVerbatim() + ", tcfProximity=" + this.getTcfProximity() + ", temporalBias=" + this.getTemporalBias() + ", temporalBiasWeight=" + this.getTemporalBiasWeight() + ", exportDebugData=" + this.isExportDebugData() + ")"; - } - - public enum TemporalBias { - RECENT, OLD, NONE - } - - public static class ResultRankingParametersBuilder { - private Bm25Parameters bm25Params; - private int shortDocumentThreshold; - private double shortDocumentPenalty; - private double domainRankBonus; - private double qualityPenalty; - private int shortSentenceThreshold; - private double shortSentencePenalty; - private double bm25Weight; - private double tcfFirstPosition; - private double tcfVerbatim; - private double tcfProximity; - private TemporalBias temporalBias; - private double temporalBiasWeight; - private boolean disablePenalties; - private boolean exportDebugData; - - ResultRankingParametersBuilder() { - } - - public ResultRankingParametersBuilder bm25Params(Bm25Parameters bm25Params) { - this.bm25Params = bm25Params; - return this; - } - - public ResultRankingParametersBuilder shortDocumentThreshold(int shortDocumentThreshold) { - this.shortDocumentThreshold = shortDocumentThreshold; - return this; - } - - public ResultRankingParametersBuilder shortDocumentPenalty(double shortDocumentPenalty) { - this.shortDocumentPenalty = shortDocumentPenalty; - return this; - } - - public ResultRankingParametersBuilder domainRankBonus(double domainRankBonus) { - this.domainRankBonus = domainRankBonus; - return this; - } - - public ResultRankingParametersBuilder qualityPenalty(double qualityPenalty) { - this.qualityPenalty = qualityPenalty; - return this; - } - - public ResultRankingParametersBuilder shortSentenceThreshold(int shortSentenceThreshold) { - this.shortSentenceThreshold = shortSentenceThreshold; - return this; - } - - public ResultRankingParametersBuilder shortSentencePenalty(double shortSentencePenalty) { - this.shortSentencePenalty = shortSentencePenalty; - return this; - } - - public ResultRankingParametersBuilder bm25Weight(double bm25Weight) { - this.bm25Weight = bm25Weight; - return this; - } - - public ResultRankingParametersBuilder tcfFirstPosition(double tcfFirstPosition) { - this.tcfFirstPosition = tcfFirstPosition; - return this; - } - - public ResultRankingParametersBuilder tcfVerbatim(double tcfVerbatim) { - this.tcfVerbatim = tcfVerbatim; - return this; - } - - public ResultRankingParametersBuilder tcfProximity(double tcfProximity) { - this.tcfProximity = tcfProximity; - return this; - } - - public ResultRankingParametersBuilder temporalBias(TemporalBias temporalBias) { - this.temporalBias = temporalBias; - return this; - } - - public ResultRankingParametersBuilder temporalBiasWeight(double temporalBiasWeight) { - this.temporalBiasWeight = temporalBiasWeight; - return this; - } - - - public ResultRankingParametersBuilder disablePenalties(boolean disablePenalties) { - this.disablePenalties = disablePenalties; - return this; - } - - public ResultRankingParametersBuilder exportDebugData(boolean exportDebugData) { - this.exportDebugData = exportDebugData; - return this; - } - - public ResultRankingParameters build() { - return new ResultRankingParameters(this.bm25Params, this.shortDocumentThreshold, this.shortDocumentPenalty, this.domainRankBonus, this.qualityPenalty, this.shortSentenceThreshold, this.shortSentencePenalty, this.bm25Weight, this.tcfFirstPosition, this.tcfVerbatim, this.tcfProximity, this.temporalBias, this.temporalBiasWeight, this.disablePenalties, this.exportDebugData); - } - - } -} diff --git a/code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java b/code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java index b7b64590..159eac8a 100644 --- a/code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java +++ b/code/functions/search-query/api/test/nu/marginalia/index/client/IndexProtobufCodecTest.java @@ -3,8 +3,6 @@ package nu.marginalia.index.client; import nu.marginalia.api.searchquery.IndexProtobufCodec; import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint; import nu.marginalia.api.searchquery.model.query.SearchQuery; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.SpecificationLimit; import org.junit.jupiter.api.Test; @@ -22,18 +20,6 @@ class IndexProtobufCodecTest { verifyIsIdentityTransformation(SpecificationLimit.lessThan(1), l -> IndexProtobufCodec.convertSpecLimit(IndexProtobufCodec.convertSpecLimit(l))); } - @Test - public void testRankingParameters() { - verifyIsIdentityTransformation(ResultRankingParameters.sensibleDefaults(), - p -> IndexProtobufCodec.convertRankingParameterss(IndexProtobufCodec.convertRankingParameterss(p, null))); - } - - @Test - public void testQueryLimits() { - verifyIsIdentityTransformation(new QueryLimits(1,2,3,4), - l -> IndexProtobufCodec.convertQueryLimits(IndexProtobufCodec.convertQueryLimits(l)) - ); - } @Test public void testSubqery() { verifyIsIdentityTransformation(new SearchQuery( diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryFactory.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryFactory.java index b0b9a8e5..a1db8cf2 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryFactory.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryFactory.java @@ -2,8 +2,9 @@ package nu.marginalia.functions.searchquery; import com.google.inject.Inject; import com.google.inject.Singleton; +import nu.marginalia.api.searchquery.RpcQueryLimits; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; import nu.marginalia.api.searchquery.model.query.*; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.functions.searchquery.query_parser.QueryExpansion; import nu.marginalia.functions.searchquery.query_parser.QueryParser; import nu.marginalia.functions.searchquery.query_parser.token.QueryToken; @@ -36,7 +37,7 @@ public class QueryFactory { public ProcessedQuery createQuery(QueryParams params, - @Nullable ResultRankingParameters rankingParams) { + @Nullable RpcResultRankingParameters rankingParams) { final var query = params.humanQuery(); if (query.length() > 1000) { @@ -132,7 +133,9 @@ public class QueryFactory { var limits = params.limits(); // Disable limits on number of results per domain if we're searching with a site:-type term if (domain != null) { - limits = limits.forSingleDomain(); + limits = RpcQueryLimits.newBuilder(limits) + .setResultsByDomain(limits.getResultsTotal()) + .build(); } var expansion = queryExpansion.expandQuery(queryBuilder.searchTermsInclude); diff --git a/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java b/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java index d17777c8..a7efc43f 100644 --- a/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java +++ b/code/functions/search-query/java/nu/marginalia/functions/searchquery/QueryGRPCService.java @@ -9,7 +9,7 @@ import nu.marginalia.api.searchquery.*; import nu.marginalia.api.searchquery.model.query.ProcessedQuery; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters; import nu.marginalia.index.api.IndexClient; import nu.marginalia.service.server.DiscoverableService; import org.slf4j.Logger; @@ -55,7 +55,7 @@ public class QueryGRPCService .time(() -> { var params = QueryProtobufCodec.convertRequest(request); - var query = queryFactory.createQuery(params, ResultRankingParameters.sensibleDefaults()); + var query = queryFactory.createQuery(params, PrototypeRankingParameters.sensibleDefaults()); var indexRequest = QueryProtobufCodec.convertQuery(request, query); @@ -102,7 +102,7 @@ public class QueryGRPCService String originalQuery, QueryParams params, IndexClient.Pagination pagination, - ResultRankingParameters rankingParameters) { + RpcResultRankingParameters rankingParameters) { var query = queryFactory.createQuery(params, rankingParameters); IndexClient.AggregateQueryResponse response = indexClient.executeQueries(QueryProtobufCodec.convertQuery(originalQuery, query), pagination); diff --git a/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java b/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java index 7bbcdfb7..e8c06501 100644 --- a/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java +++ b/code/functions/search-query/test/nu/marginalia/query/svc/QueryFactoryTest.java @@ -1,12 +1,12 @@ package nu.marginalia.query.svc; import nu.marginalia.WmsaHome; +import nu.marginalia.api.searchquery.RpcQueryLimits; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.api.searchquery.model.query.SearchSpecification; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.functions.searchquery.QueryFactory; import nu.marginalia.functions.searchquery.query_parser.QueryExpansion; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.index.query.limit.SpecificationLimitType; @@ -49,10 +49,15 @@ public class QueryFactoryTest { SpecificationLimit.none(), SpecificationLimit.none(), null, - new QueryLimits(100, 100, 100, 100), + RpcQueryLimits.newBuilder() + .setResultsTotal(100) + .setResultsByDomain(100) + .setTimeoutMs(100) + .setFetchSize(100) + .build(), "NONE", QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, 0), null).specs; } diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index 27174e48..9df3af27 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -10,12 +10,12 @@ import it.unimi.dsi.fastutil.longs.LongArrayList; import nu.marginalia.api.searchquery.IndexApiGrpc; import nu.marginalia.api.searchquery.RpcDecoratedResultItem; import nu.marginalia.api.searchquery.RpcIndexQuery; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; import nu.marginalia.api.searchquery.model.compiled.CompiledQuery; import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong; import nu.marginalia.api.searchquery.model.compiled.CqDataInt; import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.index.model.SearchParameters; @@ -211,7 +211,7 @@ public class IndexGrpcService /** This class is responsible for ranking the results and adding the best results to the * resultHeap, which depending on the state of the indexLookup threads may or may not block */ - private ResultRankingContext createRankingContext(ResultRankingParameters rankingParams, + private ResultRankingContext createRankingContext(RpcResultRankingParameters rankingParams, CompiledQuery compiledQuery, CompiledQueryLong compiledQueryIds) { diff --git a/code/index/java/nu/marginalia/index/model/SearchParameters.java b/code/index/java/nu/marginalia/index/model/SearchParameters.java index f0e851e5..2b413e50 100644 --- a/code/index/java/nu/marginalia/index/model/SearchParameters.java +++ b/code/index/java/nu/marginalia/index/model/SearchParameters.java @@ -2,16 +2,19 @@ package nu.marginalia.index.model; import nu.marginalia.api.searchquery.IndexProtobufCodec; import nu.marginalia.api.searchquery.RpcIndexQuery; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; import nu.marginalia.api.searchquery.model.compiled.CompiledQuery; import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong; import nu.marginalia.api.searchquery.model.compiled.CompiledQueryParser; -import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.query.SearchQuery; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.query.SearchSpecification; +import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters; import nu.marginalia.index.query.IndexSearchBudget; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.searchset.SearchSet; +import java.util.Objects; + import static nu.marginalia.api.searchquery.IndexProtobufCodec.convertSpecLimit; public class SearchParameters { @@ -23,7 +26,7 @@ public class SearchParameters { public final IndexSearchBudget budget; public final SearchQuery query; public final QueryParams queryParams; - public final ResultRankingParameters rankingParams; + public final RpcResultRankingParameters rankingParams; public final int limitByDomain; public final int limitTotal; @@ -41,11 +44,11 @@ public class SearchParameters { public SearchParameters(SearchSpecification specsSet, SearchSet searchSet) { var limits = specsSet.queryLimits; - this.fetchSize = limits.fetchSize(); - this.budget = new IndexSearchBudget(limits.timeoutMs()); + this.fetchSize = limits.getFetchSize(); + this.budget = new IndexSearchBudget(limits.getTimeoutMs()); this.query = specsSet.query; - this.limitByDomain = limits.resultsByDomain(); - this.limitTotal = limits.resultsTotal(); + this.limitByDomain = limits.getResultsByDomain(); + this.limitTotal = limits.getResultsTotal(); queryParams = new QueryParams( specsSet.quality, @@ -62,17 +65,17 @@ public class SearchParameters { } public SearchParameters(RpcIndexQuery request, SearchSet searchSet) { - var limits = IndexProtobufCodec.convertQueryLimits(request.getQueryLimits()); + var limits = request.getQueryLimits(); - this.fetchSize = limits.fetchSize(); + this.fetchSize = limits.getFetchSize(); // The time budget is halved because this is the point when we start to // wrap up the search and return the results. - this.budget = new IndexSearchBudget(limits.timeoutMs() / 2); + this.budget = new IndexSearchBudget(limits.getTimeoutMs() / 2); this.query = IndexProtobufCodec.convertRpcQuery(request.getQuery()); - this.limitByDomain = limits.resultsByDomain(); - this.limitTotal = limits.resultsTotal(); + this.limitByDomain = limits.getResultsByDomain(); + this.limitTotal = limits.getResultsTotal(); queryParams = new QueryParams( convertSpecLimit(request.getQuality()), @@ -85,7 +88,7 @@ public class SearchParameters { compiledQuery = CompiledQueryParser.parse(this.query.compiledQuery); compiledQueryIds = compiledQuery.mapToLong(SearchTermsUtil::getWordId); - rankingParams = IndexProtobufCodec.convertRankingParameterss(request.getParameters()); + rankingParams = Objects.requireNonNullElseGet(request.getParameters(), PrototypeRankingParameters::sensibleDefaults); } diff --git a/code/index/java/nu/marginalia/index/results/Bm25GraphVisitor.java b/code/index/java/nu/marginalia/index/results/Bm25GraphVisitor.java index 95b665ff..bf1cd12e 100644 --- a/code/index/java/nu/marginalia/index/results/Bm25GraphVisitor.java +++ b/code/index/java/nu/marginalia/index/results/Bm25GraphVisitor.java @@ -2,7 +2,6 @@ package nu.marginalia.index.results; import nu.marginalia.api.searchquery.model.compiled.CqDataInt; import nu.marginalia.api.searchquery.model.compiled.CqExpression; -import nu.marginalia.api.searchquery.model.results.Bm25Parameters; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import java.util.BitSet; @@ -24,14 +23,14 @@ public class Bm25GraphVisitor implements CqExpression.DoubleVisitor { private final BitSet mask; - public Bm25GraphVisitor(Bm25Parameters bm25Parameters, + public Bm25GraphVisitor(double k1, double b, float[] counts, int length, ResultRankingContext ctx) { this.length = length; - this.k1 = bm25Parameters.k(); - this.b = bm25Parameters.b(); + this.k1 = k1; + this.b = b; this.docCount = ctx.termFreqDocCount(); this.counts = counts; diff --git a/code/index/java/nu/marginalia/index/results/IndexResultRankingService.java b/code/index/java/nu/marginalia/index/results/IndexResultRankingService.java index 91654052..9920b4da 100644 --- a/code/index/java/nu/marginalia/index/results/IndexResultRankingService.java +++ b/code/index/java/nu/marginalia/index/results/IndexResultRankingService.java @@ -156,7 +156,7 @@ public class IndexResultRankingService { // for the selected results, as this would be comically expensive to do for all the results we // discard along the way - if (params.rankingParams.exportDebugData) { + if (params.rankingParams.getExportDebugData()) { var combinedIdsList = new LongArrayList(resultsList.size()); for (var item : resultsList) { combinedIdsList.add(item.combinedId); diff --git a/code/index/java/nu/marginalia/index/results/IndexResultScoreCalculator.java b/code/index/java/nu/marginalia/index/results/IndexResultScoreCalculator.java index ab32e66c..72e77a19 100644 --- a/code/index/java/nu/marginalia/index/results/IndexResultScoreCalculator.java +++ b/code/index/java/nu/marginalia/index/results/IndexResultScoreCalculator.java @@ -2,10 +2,11 @@ package nu.marginalia.index.results; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntList; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.api.searchquery.model.compiled.CompiledQuery; import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; import nu.marginalia.api.searchquery.model.results.SearchResultItem; import nu.marginalia.api.searchquery.model.results.debug.DebugRankingFactors; import nu.marginalia.index.forward.spans.DocumentSpans; @@ -116,14 +117,14 @@ public class IndexResultScoreCalculator { float proximitiyFac = getProximitiyFac(decodedPositions, searchTerms.phraseConstraints, verbatimMatches, unorderedMatches, spans); - double score_firstPosition = params.tcfFirstPosition * (1.0 / Math.sqrt(unorderedMatches.firstPosition)); - double score_verbatim = params.tcfVerbatim * verbatimMatches.getScore(); - double score_proximity = params.tcfProximity * proximitiyFac; - double score_bM25 = params.bm25Weight - * wordFlagsQuery.root.visit(new Bm25GraphVisitor(params.bm25Params, unorderedMatches.getWeightedCounts(), docSize, rankingContext)) + double score_firstPosition = params.getTcfFirstPositionWeight() * (1.0 / Math.sqrt(unorderedMatches.firstPosition)); + double score_verbatim = params.getTcfVerbatimWeight() * verbatimMatches.getScore(); + double score_proximity = params.getTcfProximityWeight() * proximitiyFac; + double score_bM25 = params.getBm25Weight() + * wordFlagsQuery.root.visit(new Bm25GraphVisitor(params.getBm25K(), params.getBm25B(), unorderedMatches.getWeightedCounts(), docSize, rankingContext)) / (Math.sqrt(unorderedMatches.searchableKeywordCount + 1)); - double score_bFlags = params.bm25Weight - * wordFlagsQuery.root.visit(new TermFlagsGraphVisitor(params.bm25Params, wordFlagsQuery.data, unorderedMatches.getWeightedCounts(), rankingContext)) + double score_bFlags = params.getBm25Weight() + * wordFlagsQuery.root.visit(new TermFlagsGraphVisitor(params.getBm25K(), wordFlagsQuery.data, unorderedMatches.getWeightedCounts(), rankingContext)) / (Math.sqrt(unorderedMatches.searchableKeywordCount + 1)); double score = normalize( @@ -245,10 +246,10 @@ public class IndexResultScoreCalculator { private double calculateDocumentBonus(long documentMetadata, int features, int length, - ResultRankingParameters rankingParams, + RpcResultRankingParameters rankingParams, @Nullable DebugRankingFactors debugRankingFactors) { - if (rankingParams.disablePenalties) { + if (rankingParams.getDisablePenalties()) { return 0.; } @@ -260,18 +261,18 @@ public class IndexResultScoreCalculator { int topology = DocumentMetadata.decodeTopology(documentMetadata); int year = DocumentMetadata.decodeYear(documentMetadata); - double averageSentenceLengthPenalty = (asl >= rankingParams.shortSentenceThreshold ? 0 : -rankingParams.shortSentencePenalty); + double averageSentenceLengthPenalty = (asl >= rankingParams.getShortSentenceThreshold() ? 0 : -rankingParams.getShortSentencePenalty()); final double qualityPenalty = calculateQualityPenalty(size, quality, rankingParams); - final double rankingBonus = (255. - rank) * rankingParams.domainRankBonus; + final double rankingBonus = (255. - rank) * rankingParams.getDomainRankBonus(); final double topologyBonus = Math.log(1 + topology); - final double documentLengthPenalty = length > rankingParams.shortDocumentThreshold ? 0 : -rankingParams.shortDocumentPenalty; + final double documentLengthPenalty = length > rankingParams.getShortDocumentThreshold() ? 0 : -rankingParams.getShortDocumentPenalty(); final double temporalBias; - if (rankingParams.temporalBias == ResultRankingParameters.TemporalBias.RECENT) { - temporalBias = - Math.abs(year - PubDate.MAX_YEAR) * rankingParams.temporalBiasWeight; - } else if (rankingParams.temporalBias == ResultRankingParameters.TemporalBias.OLD) { - temporalBias = - Math.abs(year - PubDate.MIN_YEAR) * rankingParams.temporalBiasWeight; + if (rankingParams.getTemporalBias().getBias() == RpcTemporalBias.Bias.RECENT) { + temporalBias = - Math.abs(year - PubDate.MAX_YEAR) * rankingParams.getTemporalBiasWeight(); + } else if (rankingParams.getTemporalBias().getBias() == RpcTemporalBias.Bias.OLD) { + temporalBias = - Math.abs(year - PubDate.MIN_YEAR) * rankingParams.getTemporalBiasWeight(); } else { temporalBias = 0; } @@ -510,14 +511,14 @@ public class IndexResultScoreCalculator { } - private double calculateQualityPenalty(int size, int quality, ResultRankingParameters rankingParams) { + private double calculateQualityPenalty(int size, int quality, RpcResultRankingParameters rankingParams) { if (size < 400) { if (quality < 5) return 0; - return -quality * rankingParams.qualityPenalty; + return -quality * rankingParams.getQualityPenalty(); } else { - return -quality * rankingParams.qualityPenalty * 20; + return -quality * rankingParams.getQualityPenalty() * 20; } } diff --git a/code/index/java/nu/marginalia/index/results/TermFlagsGraphVisitor.java b/code/index/java/nu/marginalia/index/results/TermFlagsGraphVisitor.java index e4255a5e..d4b9e187 100644 --- a/code/index/java/nu/marginalia/index/results/TermFlagsGraphVisitor.java +++ b/code/index/java/nu/marginalia/index/results/TermFlagsGraphVisitor.java @@ -3,7 +3,6 @@ package nu.marginalia.index.results; import nu.marginalia.api.searchquery.model.compiled.CqDataInt; import nu.marginalia.api.searchquery.model.compiled.CqDataLong; import nu.marginalia.api.searchquery.model.compiled.CqExpression; -import nu.marginalia.api.searchquery.model.results.Bm25Parameters; import nu.marginalia.api.searchquery.model.results.ResultRankingContext; import nu.marginalia.model.idx.WordFlags; @@ -15,15 +14,14 @@ public class TermFlagsGraphVisitor implements CqExpression.DoubleVisitor { private final CqDataLong wordMetaData; private final CqDataInt frequencies; private final float[] counts; - private final Bm25Parameters bm25Parameters; - + private final double k1; private final int docCount; - public TermFlagsGraphVisitor(Bm25Parameters bm25Parameters, + public TermFlagsGraphVisitor(double k1, CqDataLong wordMetaData, float[] counts, ResultRankingContext ctx) { - this.bm25Parameters = bm25Parameters; + this.k1 = k1; this.counts = counts; this.docCount = ctx.termFreqDocCount(); this.wordMetaData = wordMetaData; @@ -55,7 +53,7 @@ public class TermFlagsGraphVisitor implements CqExpression.DoubleVisitor { int freq = frequencies.get(idx); // note we override b to zero for priority terms as they are independent of document length - return invFreq(docCount, freq) * f(bm25Parameters.k(), 0, count, 0); + return invFreq(docCount, freq) * f(k1, 0, count, 0); } private double evaluatePriorityScore(int idx) { diff --git a/code/index/query/java/nu/marginalia/index/query/limit/QueryLimits.java b/code/index/query/java/nu/marginalia/index/query/limit/QueryLimits.java deleted file mode 100644 index 32e397fc..00000000 --- a/code/index/query/java/nu/marginalia/index/query/limit/QueryLimits.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.index.query.limit; - -public record QueryLimits(int resultsByDomain, int resultsTotal, int timeoutMs, int fetchSize) { - public QueryLimits forSingleDomain() { - return new QueryLimits(resultsTotal, resultsTotal, timeoutMs, fetchSize); - } -} diff --git a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java index 7135378e..2ff2a5da 100644 --- a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java +++ b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationSmokeTest.java @@ -4,10 +4,11 @@ import com.google.inject.Guice; import com.google.inject.Inject; import nu.marginalia.IndexLocations; import nu.marginalia.api.searchquery.RpcDecoratedResultItem; +import nu.marginalia.api.searchquery.RpcQueryLimits; import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint; import nu.marginalia.api.searchquery.model.query.SearchQuery; import nu.marginalia.api.searchquery.model.query.SearchSpecification; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters; import nu.marginalia.index.construction.DocIdRewriter; import nu.marginalia.index.construction.full.FullIndexConstructor; import nu.marginalia.index.construction.prio.PrioIndexConstructor; @@ -17,7 +18,6 @@ import nu.marginalia.index.forward.construction.ForwardIndexConverter; import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.index.journal.IndexJournal; import nu.marginalia.index.journal.IndexJournalSlopWriter; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.linkdb.docs.DocumentDbReader; @@ -115,9 +115,16 @@ public class IndexQueryServiceIntegrationSmokeTest { var rsp = queryService.justQuery( SearchSpecification.builder() - .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) + .queryLimits( + RpcQueryLimits.newBuilder() + .setResultsByDomain(10) + .setResultsTotal(10) + .setTimeoutMs(Integer.MAX_VALUE) + .setFetchSize(4000) + .build() + ) .queryStrategy(QueryStrategy.SENTENCE) - .rankingParams(ResultRankingParameters.sensibleDefaults()) + .rankingParams(PrototypeRankingParameters.sensibleDefaults()) .domains(new ArrayList<>()) .searchSetIdentifier("NONE") .query( @@ -171,9 +178,16 @@ public class IndexQueryServiceIntegrationSmokeTest { var rsp = queryService.justQuery( SearchSpecification.builder() - .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) + .queryLimits( + RpcQueryLimits.newBuilder() + .setResultsByDomain(10) + .setResultsTotal(10) + .setTimeoutMs(Integer.MAX_VALUE) + .setFetchSize(4000) + .build() + ) .queryStrategy(QueryStrategy.SENTENCE) - .rankingParams(ResultRankingParameters.sensibleDefaults()) + .rankingParams(PrototypeRankingParameters.sensibleDefaults()) .domains(new ArrayList<>()) .searchSetIdentifier("NONE") .query( @@ -225,8 +239,15 @@ public class IndexQueryServiceIntegrationSmokeTest { var rsp = queryService.justQuery( SearchSpecification.builder() - .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) - .rankingParams(ResultRankingParameters.sensibleDefaults()) + .queryLimits( + RpcQueryLimits.newBuilder() + .setResultsByDomain(10) + .setResultsTotal(10) + .setTimeoutMs(Integer.MAX_VALUE) + .setFetchSize(4000) + .build() + ) + .rankingParams(PrototypeRankingParameters.sensibleDefaults()) .queryStrategy(QueryStrategy.SENTENCE) .domains(List.of(2)) .query( @@ -282,11 +303,18 @@ public class IndexQueryServiceIntegrationSmokeTest { var rsp = queryService.justQuery( SearchSpecification.builder() - .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) + .queryLimits( + RpcQueryLimits.newBuilder() + .setResultsByDomain(10) + .setResultsTotal(10) + .setTimeoutMs(Integer.MAX_VALUE) + .setFetchSize(4000) + .build() + ) .year(SpecificationLimit.equals(1998)) .queryStrategy(QueryStrategy.SENTENCE) .searchSetIdentifier("NONE") - .rankingParams(ResultRankingParameters.sensibleDefaults()) + .rankingParams(PrototypeRankingParameters.sensibleDefaults()) .query( SearchQuery.builder() .compiledQuery("4") diff --git a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java index ed167080..6b23d78a 100644 --- a/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java +++ b/code/index/test/nu/marginalia/index/IndexQueryServiceIntegrationTest.java @@ -4,10 +4,11 @@ import com.google.inject.Guice; import com.google.inject.Inject; import it.unimi.dsi.fastutil.ints.IntList; import nu.marginalia.IndexLocations; +import nu.marginalia.api.searchquery.RpcQueryLimits; import nu.marginalia.api.searchquery.model.query.SearchPhraseConstraint; import nu.marginalia.api.searchquery.model.query.SearchQuery; import nu.marginalia.api.searchquery.model.query.SearchSpecification; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters; import nu.marginalia.hash.MurmurHash3_128; import nu.marginalia.index.construction.DocIdRewriter; import nu.marginalia.index.construction.full.FullIndexConstructor; @@ -18,7 +19,6 @@ import nu.marginalia.index.forward.construction.ForwardIndexConverter; import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.index.journal.IndexJournal; import nu.marginalia.index.journal.IndexJournalSlopWriter; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.linkdb.docs.DocumentDbReader; @@ -389,13 +389,20 @@ public class IndexQueryServiceIntegrationTest { SearchSpecification basicQuery(Function mutator) { var builder = SearchSpecification.builder() - .queryLimits(new QueryLimits(10, 10, Integer.MAX_VALUE, 4000)) + .queryLimits( + RpcQueryLimits.newBuilder() + .setResultsByDomain(10) + .setResultsTotal(10) + .setTimeoutMs(Integer.MAX_VALUE) + .setFetchSize(4000) + .build() + ) .queryStrategy(QueryStrategy.SENTENCE) .year(SpecificationLimit.none()) .quality(SpecificationLimit.none()) .size(SpecificationLimit.none()) .rank(SpecificationLimit.none()) - .rankingParams(ResultRankingParameters.sensibleDefaults()) + .rankingParams(PrototypeRankingParameters.sensibleDefaults()) .domains(new ArrayList<>()) .searchSetIdentifier("NONE"); diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index 0680f59e..c88084f5 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -44,6 +44,7 @@ dependencies { implementation libs.bundles.jetty implementation libs.opencsv implementation libs.trove + implementation libs.protobuf implementation libs.fastutil implementation libs.bundles.gson implementation libs.bundles.mariadb diff --git a/code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java b/code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java index 95145de3..2c0286b6 100644 --- a/code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java +++ b/code/services-application/api-service/java/nu/marginalia/api/ApiSearchOperator.java @@ -6,10 +6,10 @@ import nu.marginalia.api.model.ApiSearchResult; import nu.marginalia.api.model.ApiSearchResultQueryDetails; import nu.marginalia.api.model.ApiSearchResults; import nu.marginalia.api.searchquery.QueryClient; +import nu.marginalia.api.searchquery.RpcQueryLimits; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.idx.WordFlags; import java.util.ArrayList; @@ -47,11 +47,12 @@ public class ApiSearchOperator { return new QueryParams( query, - new QueryLimits( - 2, - Math.min(100, count), - 150, - 8192), + RpcQueryLimits.newBuilder() + .setResultsByDomain(2) + .setResultsTotal(Math.min(100, count)) + .setTimeoutMs(150) + .setFetchSize(8192) + .build(), searchSet.name()); } diff --git a/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchOperator.java index 9a86db64..8a6f5b1d 100644 --- a/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchOperator.java +++ b/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchOperator.java @@ -5,11 +5,11 @@ import com.google.inject.Singleton; import nu.marginalia.WebsiteUrl; import nu.marginalia.api.math.MathClient; import nu.marginalia.api.searchquery.QueryClient; +import nu.marginalia.api.searchquery.RpcQueryLimits; import nu.marginalia.api.searchquery.model.query.QueryResponse; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.bbpc.BrailleBlockPunchCards; import nu.marginalia.db.DbDomainQueries; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.EdgeUrl; import nu.marginalia.model.crawl.DomainIndexingState; @@ -155,15 +155,15 @@ public class SearchOperator { public List getResultsFromQuery(QueryResponse queryResponse) { - final QueryLimits limits = queryResponse.specs().queryLimits; - final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.resultsByDomain()); + final RpcQueryLimits limits = queryResponse.specs().queryLimits; + final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.getResultsByDomain()); // Update the query count (this is what you see on the front page) searchVisitorCount.registerQuery(); return queryResponse.results().stream() .filter(deduplicator::shouldRetain) - .limit(limits.resultsTotal()) + .limit(limits.getResultsTotal()) .map(SearchOperator::createDetails) .toList(); } diff --git a/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchQueryParamFactory.java b/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchQueryParamFactory.java index 6852423a..db2f7f84 100644 --- a/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchQueryParamFactory.java +++ b/code/services-application/search-service-legacy/java/nu/marginalia/search/SearchQueryParamFactory.java @@ -1,10 +1,10 @@ package nu.marginalia.search; +import nu.marginalia.api.searchquery.RpcQueryLimits; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.api.searchquery.model.query.SearchQuery; import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.search.command.SearchParameters; @@ -12,6 +12,21 @@ import nu.marginalia.search.command.SearchParameters; import java.util.List; public class SearchQueryParamFactory { + static final RpcQueryLimits defaultLimits = RpcQueryLimits.newBuilder() + .setResultsTotal(100) + .setResultsByDomain(5) + .setTimeoutMs(200) + .setFetchSize(8192) + .build(); + + + static final RpcQueryLimits shallowLimit = RpcQueryLimits.newBuilder() + .setResultsTotal(100) + .setResultsByDomain(100) + .setTimeoutMs(100) + .setFetchSize(512) + .build(); + public QueryParams forRegularSearch(SearchParameters userParams) { SearchQuery prototype = new SearchQuery(); @@ -33,7 +48,7 @@ public class SearchQueryParamFactory { profile.getSizeLimit(), SpecificationLimit.none(), List.of(), - new QueryLimits(5, 100, 200, 8192), + defaultLimits, profile.searchSetIdentifier.name(), userParams.strategy(), userParams.temporalBias(), @@ -54,10 +69,15 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), List.of(domainId), - new QueryLimits(count, count, 100, 512), + RpcQueryLimits.newBuilder() + .setResultsTotal(count) + .setResultsByDomain(count) + .setTimeoutMs(100) + .setFetchSize(512) + .build(), SearchSetIdentifier.NONE.name(), QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, 1 ); } @@ -74,10 +94,10 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), List.of(), - new QueryLimits(100, 100, 100, 512), + shallowLimit, SearchSetIdentifier.NONE.name(), QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, 1 ); } @@ -94,10 +114,10 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), List.of(), - new QueryLimits(100, 100, 100, 512), + shallowLimit, SearchSetIdentifier.NONE.name(), QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, 1 ); } diff --git a/code/services-application/search-service-legacy/java/nu/marginalia/search/command/SearchParameters.java b/code/services-application/search-service-legacy/java/nu/marginalia/search/command/SearchParameters.java index c10d0092..0e2bbb13 100644 --- a/code/services-application/search-service-legacy/java/nu/marginalia/search/command/SearchParameters.java +++ b/code/services-application/search-service-legacy/java/nu/marginalia/search/command/SearchParameters.java @@ -1,7 +1,7 @@ package nu.marginalia.search.command; import nu.marginalia.WebsiteUrl; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.search.model.SearchProfile; @@ -78,15 +78,15 @@ public record SearchParameters(String query, return baseUrl.withPath(path); } - public ResultRankingParameters.TemporalBias temporalBias() { + public RpcTemporalBias.Bias temporalBias() { if (recent == RECENT) { - return ResultRankingParameters.TemporalBias.RECENT; + return RpcTemporalBias.Bias.RECENT; } else if (profile == SearchProfile.VINTAGE) { - return ResultRankingParameters.TemporalBias.OLD; + return RpcTemporalBias.Bias.OLD; } - return ResultRankingParameters.TemporalBias.NONE; + return RpcTemporalBias.Bias.NONE; } public QueryStrategy strategy() { diff --git a/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java index 8acc297e..b2588472 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java +++ b/code/services-application/search-service/java/nu/marginalia/search/SearchOperator.java @@ -2,14 +2,13 @@ package nu.marginalia.search; import com.google.inject.Inject; import com.google.inject.Singleton; -import nu.marginalia.WebsiteUrl; import nu.marginalia.api.math.MathClient; import nu.marginalia.api.searchquery.QueryClient; +import nu.marginalia.api.searchquery.RpcQueryLimits; import nu.marginalia.api.searchquery.model.query.QueryResponse; import nu.marginalia.api.searchquery.model.results.DecoratedSearchResultItem; import nu.marginalia.bbpc.BrailleBlockPunchCards; import nu.marginalia.db.DbDomainQueries; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.EdgeUrl; import nu.marginalia.model.crawl.DomainIndexingState; @@ -47,7 +46,6 @@ public class SearchOperator { private final DbDomainQueries domainQueries; private final QueryClient queryClient; private final SearchQueryParamFactory paramFactory; - private final WebsiteUrl websiteUrl; private final SearchUnitConversionService searchUnitConversionService; private final SearchQueryCountService searchVisitorCount; @@ -57,7 +55,6 @@ public class SearchOperator { DbDomainQueries domainQueries, QueryClient queryClient, SearchQueryParamFactory paramFactory, - WebsiteUrl websiteUrl, SearchUnitConversionService searchUnitConversionService, SearchQueryCountService searchVisitorCount ) @@ -67,7 +64,6 @@ public class SearchOperator { this.domainQueries = domainQueries; this.queryClient = queryClient; this.paramFactory = paramFactory; - this.websiteUrl = websiteUrl; this.searchUnitConversionService = searchUnitConversionService; this.searchVisitorCount = searchVisitorCount; } @@ -154,8 +150,8 @@ public class SearchOperator { public SimpleSearchResults getResultsFromQuery(QueryResponse queryResponse) { - final QueryLimits limits = queryResponse.specs().queryLimits; - final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.resultsByDomain()); + final RpcQueryLimits limits = queryResponse.specs().queryLimits; + final UrlDeduplicator deduplicator = new UrlDeduplicator(limits.getResultsByDomain()); // Update the query count (this is what you see on the front page) searchVisitorCount.registerQuery(); @@ -164,7 +160,7 @@ public class SearchOperator { .sorted(this::retentionSortOrder) // Sort in an order that makes us more likely to discard the "bad" duplicates .filter(deduplicator::shouldRetain) .sorted() // Return to the presentation sort order before limiting so we don't throw out good results over schema and "ip-ness" - .limit(limits.resultsTotal()) + .limit(limits.getResultsTotal()) .map(SearchOperator::createDetails) .toList(); diff --git a/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java b/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java index 3758730f..23f4d15b 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java +++ b/code/services-application/search-service/java/nu/marginalia/search/SearchQueryParamFactory.java @@ -1,10 +1,10 @@ package nu.marginalia.search; +import nu.marginalia.api.searchquery.RpcQueryLimits; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.api.searchquery.model.query.SearchQuery; import nu.marginalia.api.searchquery.model.query.SearchSetIdentifier; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.search.command.SearchParameters; @@ -13,6 +13,22 @@ import java.util.List; public class SearchQueryParamFactory { + + static final RpcQueryLimits defaultLimits = RpcQueryLimits.newBuilder() + .setResultsTotal(100) + .setResultsByDomain(5) + .setTimeoutMs(200) + .setFetchSize(8192) + .build(); + + + static final RpcQueryLimits shallowLimit = RpcQueryLimits.newBuilder() + .setResultsTotal(100) + .setResultsByDomain(100) + .setTimeoutMs(100) + .setFetchSize(512) + .build(); + public QueryParams forRegularSearch(SearchParameters userParams) { SearchQuery prototype = new SearchQuery(); var profile = userParams.profile(); @@ -29,11 +45,11 @@ public class SearchQueryParamFactory { prototype.searchTermsPriority, prototype.searchTermsAdvice, profile.getQualityLimit(), - profile.getYearLimit(), + userParams.yearLimit(), profile.getSizeLimit(), SpecificationLimit.none(), List.of(), - new QueryLimits(5, 100, 200, 8192), + defaultLimits, profile.searchSetIdentifier.name(), userParams.strategy(), userParams.temporalBias(), @@ -54,10 +70,15 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), List.of(domainId), - new QueryLimits(count, count, 100, 512), + RpcQueryLimits.newBuilder() + .setResultsTotal(count) + .setResultsByDomain(count) + .setTimeoutMs(100) + .setFetchSize(512) + .build(), SearchSetIdentifier.NONE.name(), QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, page ); } @@ -74,10 +95,10 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), List.of(), - new QueryLimits(100, 100, 100, 512), + shallowLimit, SearchSetIdentifier.NONE.name(), QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, page ); } @@ -94,10 +115,10 @@ public class SearchQueryParamFactory { SpecificationLimit.none(), SpecificationLimit.none(), List.of(), - new QueryLimits(100, 100, 100, 512), + shallowLimit, SearchSetIdentifier.NONE.name(), QueryStrategy.AUTO, - ResultRankingParameters.TemporalBias.NONE, + RpcTemporalBias.Bias.NONE, 1 ); } diff --git a/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java b/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java index 57f79ae0..d507c44c 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java +++ b/code/services-application/search-service/java/nu/marginalia/search/command/SearchParameters.java @@ -1,7 +1,7 @@ package nu.marginalia.search.command; import nu.marginalia.WebsiteUrl; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; import nu.marginalia.model.EdgeDomain; @@ -98,15 +98,15 @@ public record SearchParameters(WebsiteUrl url, return path; } - public ResultRankingParameters.TemporalBias temporalBias() { + public RpcTemporalBias.Bias temporalBias() { if (recent == RECENT) { - return ResultRankingParameters.TemporalBias.RECENT; + return RpcTemporalBias.Bias.RECENT; } else if (profile == SearchProfile.VINTAGE) { - return ResultRankingParameters.TemporalBias.OLD; + return RpcTemporalBias.Bias.OLD; } - return ResultRankingParameters.TemporalBias.NONE; + return RpcTemporalBias.Bias.NONE; } public QueryStrategy strategy() { diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index 20db2bed..8f216ed6 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -55,6 +55,7 @@ dependencies { implementation libs.duckdb implementation libs.jsoup + implementation libs.protobuf implementation libs.trove implementation dependencies.create(libs.spark.get()) { diff --git a/code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java index 7993009c..4171fcb1 100644 --- a/code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java +++ b/code/services-core/control-service/java/nu/marginalia/control/app/svc/SearchToBanService.java @@ -2,11 +2,10 @@ package nu.marginalia.control.app.svc; import com.google.inject.Inject; import nu.marginalia.api.searchquery.QueryClient; +import nu.marginalia.api.searchquery.RpcQueryLimits; import nu.marginalia.api.searchquery.model.query.QueryParams; import nu.marginalia.control.ControlRendererFactory; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.EdgeUrl; -import nu.marginalia.nodecfg.NodeConfigurationService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -22,18 +21,16 @@ public class SearchToBanService { private final ControlRendererFactory rendererFactory; private final QueryClient queryClient; private final Logger logger = LoggerFactory.getLogger(getClass()); - private final NodeConfigurationService nodeConfigurationService; @Inject public SearchToBanService(ControlBlacklistService blacklistService, ControlRendererFactory rendererFactory, - QueryClient queryClient, NodeConfigurationService nodeConfigurationService) + QueryClient queryClient) { this.blacklistService = blacklistService; this.rendererFactory = rendererFactory; this.queryClient = queryClient; - this.nodeConfigurationService = nodeConfigurationService; } public void register() throws IOException { @@ -76,7 +73,14 @@ public class SearchToBanService { private Object executeQuery(String query) { return queryClient.search(new QueryParams( - query, new QueryLimits(2, 200, 250, 8192), + query, + RpcQueryLimits.newBuilder() + .setResultsTotal(100) + .setResultsByDomain(2) + .setTimeoutMs(200) + .setFetchSize(8192) + .build() + , "NONE" )); } diff --git a/code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java b/code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java index f32785df..dbe056d8 100644 --- a/code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java +++ b/code/services-core/query-service/java/nu/marginalia/query/QueryBasicInterface.java @@ -3,12 +3,13 @@ package nu.marginalia.query; import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.inject.Inject; +import nu.marginalia.api.searchquery.RpcQueryLimits; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; +import nu.marginalia.api.searchquery.RpcTemporalBias; import nu.marginalia.api.searchquery.model.query.QueryParams; -import nu.marginalia.api.searchquery.model.results.Bm25Parameters; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters; import nu.marginalia.functions.searchquery.QueryGRPCService; import nu.marginalia.index.api.IndexClient; -import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; @@ -53,9 +54,14 @@ public class QueryBasicInterface { int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5")); String set = requireNonNullElse(request.queryParams("set"), ""); - var params = new QueryParams(queryString, new QueryLimits( - domainCount, min(100, count * 10), 250, 8192 - ), set); + var params = new QueryParams(queryString, + RpcQueryLimits.newBuilder() + .setResultsByDomain(domainCount) + .setResultsTotal(min(100, count * 10)) + .setTimeoutMs(250) + .setFetchSize(8192) + .build() + , set); var pagination = new IndexClient.Pagination(page, count); @@ -63,7 +69,7 @@ public class QueryBasicInterface { queryString, params, pagination, - ResultRankingParameters.sensibleDefaults() + PrototypeRankingParameters.sensibleDefaults() ); var results = detailedDirectResult.result(); @@ -92,7 +98,7 @@ public class QueryBasicInterface { String queryString = request.queryParams("q"); if (queryString == null) { // Show the default query form if no query is given - return qdebugRenderer.render(Map.of("rankingParams", ResultRankingParameters.sensibleDefaults()) + return qdebugRenderer.render(Map.of("rankingParams", PrototypeRankingParameters.sensibleDefaults()) ); } @@ -101,9 +107,14 @@ public class QueryBasicInterface { int domainCount = parseInt(requireNonNullElse(request.queryParams("domainCount"), "5")); String set = requireNonNullElse(request.queryParams("set"), ""); - var queryParams = new QueryParams(queryString, new QueryLimits( - domainCount, min(100, count * 10), 250, 8192 - ), set); + var queryParams = new QueryParams(queryString, + RpcQueryLimits.newBuilder() + .setResultsByDomain(domainCount) + .setResultsTotal(min(100, count * 10)) + .setTimeoutMs(250) + .setFetchSize(8192) + .build(), + set); var pagination = new IndexClient.Pagination(page, count); @@ -126,28 +137,28 @@ public class QueryBasicInterface { ); } - private ResultRankingParameters debugRankingParamsFromRequest(Request request) { - var sensibleDefaults = ResultRankingParameters.sensibleDefaults(); + private RpcResultRankingParameters debugRankingParamsFromRequest(Request request) { + var sensibleDefaults = PrototypeRankingParameters.sensibleDefaults(); - return ResultRankingParameters.builder() - .domainRankBonus(doubleFromRequest(request, "domainRankBonus", sensibleDefaults.domainRankBonus)) - .qualityPenalty(doubleFromRequest(request, "qualityPenalty", sensibleDefaults.qualityPenalty)) - .shortDocumentThreshold(intFromRequest(request, "shortDocumentThreshold", sensibleDefaults.shortDocumentThreshold)) - .shortDocumentPenalty(doubleFromRequest(request, "shortDocumentPenalty", sensibleDefaults.shortDocumentPenalty)) - .tcfFirstPosition(doubleFromRequest(request, "tcfFirstPosition", sensibleDefaults.tcfFirstPosition)) - .tcfVerbatim(doubleFromRequest(request, "tcfVerbatim", sensibleDefaults.tcfVerbatim)) - .tcfProximity(doubleFromRequest(request, "tcfProximity", sensibleDefaults.tcfProximity)) - .bm25Params(new Bm25Parameters( - doubleFromRequest(request, "bm25.k1", sensibleDefaults.bm25Params.k()), - doubleFromRequest(request, "bm25.b", sensibleDefaults.bm25Params.b()) - )) - .temporalBias(ResultRankingParameters.TemporalBias.valueOf(stringFromRequest(request, "temporalBias", sensibleDefaults.temporalBias.toString()))) - .temporalBiasWeight(doubleFromRequest(request, "temporalBiasWeight", sensibleDefaults.temporalBiasWeight)) - .shortSentenceThreshold(intFromRequest(request, "shortSentenceThreshold", sensibleDefaults.shortSentenceThreshold)) - .shortSentencePenalty(doubleFromRequest(request, "shortSentencePenalty", sensibleDefaults.shortSentencePenalty)) - .bm25Weight(doubleFromRequest(request, "bm25Weight", sensibleDefaults.bm25Weight)) - .disablePenalties(boolFromRequest(request, "disablePenalties", sensibleDefaults.disablePenalties)) - .exportDebugData(true) + var bias = RpcTemporalBias.Bias.valueOf(stringFromRequest(request, "temporalBias", "NONE")); + + return RpcResultRankingParameters.newBuilder() + .setDomainRankBonus(doubleFromRequest(request, "domainRankBonus", sensibleDefaults.getDomainRankBonus())) + .setQualityPenalty(doubleFromRequest(request, "qualityPenalty", sensibleDefaults.getQualityPenalty())) + .setShortDocumentThreshold(intFromRequest(request, "shortDocumentThreshold", sensibleDefaults.getShortDocumentThreshold())) + .setShortDocumentPenalty(doubleFromRequest(request, "shortDocumentPenalty", sensibleDefaults.getShortDocumentPenalty())) + .setTcfFirstPositionWeight(doubleFromRequest(request, "tcfFirstPositionWeight", sensibleDefaults.getTcfFirstPositionWeight())) + .setTcfVerbatimWeight(doubleFromRequest(request, "tcfVerbatimWeight", sensibleDefaults.getTcfVerbatimWeight())) + .setTcfProximityWeight(doubleFromRequest(request, "tcfProximityWeight", sensibleDefaults.getTcfProximityWeight())) + .setBm25B(doubleFromRequest(request, "bm25b", sensibleDefaults.getBm25B())) + .setBm25K(doubleFromRequest(request, "bm25k", sensibleDefaults.getBm25K())) + .setTemporalBias(RpcTemporalBias.newBuilder().setBias(bias).build()) + .setTemporalBiasWeight(doubleFromRequest(request, "temporalBiasWeight", sensibleDefaults.getTemporalBiasWeight())) + .setShortSentenceThreshold(intFromRequest(request, "shortSentenceThreshold", sensibleDefaults.getShortSentenceThreshold())) + .setShortSentencePenalty(doubleFromRequest(request, "shortSentencePenalty", sensibleDefaults.getShortSentencePenalty())) + .setBm25Weight(doubleFromRequest(request, "bm25Weight", sensibleDefaults.getBm25Weight())) + .setDisablePenalties(boolFromRequest(request, "disablePenalties", sensibleDefaults.getDisablePenalties())) + .setExportDebugData(true) .build(); } diff --git a/code/services-core/query-service/resources/templates/qdebug.hdb b/code/services-core/query-service/resources/templates/qdebug.hdb index 05dcb6d7..50bbf92e 100644 --- a/code/services-core/query-service/resources/templates/qdebug.hdb +++ b/code/services-core/query-service/resources/templates/qdebug.hdb @@ -31,20 +31,20 @@
-
-
+
+
-
-
-
-
+
+
+
+
-
-
-
-
+
+
+
+
diff --git a/code/tools/integration-test/test/nu/marginalia/IntegrationTest.java b/code/tools/integration-test/test/nu/marginalia/IntegrationTest.java index 498daa79..1adf5678 100644 --- a/code/tools/integration-test/test/nu/marginalia/IntegrationTest.java +++ b/code/tools/integration-test/test/nu/marginalia/IntegrationTest.java @@ -5,7 +5,8 @@ import com.google.inject.Inject; import nu.marginalia.api.searchquery.QueryProtobufCodec; import nu.marginalia.api.searchquery.RpcQsQuery; import nu.marginalia.api.searchquery.RpcQueryLimits; -import nu.marginalia.api.searchquery.model.results.ResultRankingParameters; +import nu.marginalia.api.searchquery.RpcResultRankingParameters; +import nu.marginalia.api.searchquery.model.results.PrototypeRankingParameters; import nu.marginalia.converting.processor.DomainProcessor; import nu.marginalia.converting.writer.ConverterBatchWriter; import nu.marginalia.crawl.fetcher.ContentTags; @@ -211,8 +212,7 @@ public class IntegrationTest { var params = QueryProtobufCodec.convertRequest(request); - var p = ResultRankingParameters.sensibleDefaults(); - p.exportDebugData = true; + var p = RpcResultRankingParameters.newBuilder(PrototypeRankingParameters.sensibleDefaults()).setExportDebugData(true).build(); var query = queryFactory.createQuery(params, p);