Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(qs) Additional info in query debug UI

Browse Source
This commit is contained in:
Viktor Lofgren 2024-04-19 12:18:53 +02:00
parent eb74d08f2a
commit b80a83339b
8 changed files with 88 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
code/functions/search-query/api/java/nu/marginalia/api/searchquery/IndexProtobufCodec.java
View File

@ -3,6 +3,9 @@ package nu.marginalia.api.searchquery;
import nu.marginalia.api.searchquery.model.query.SearchQuery;
import nu.marginalia.api.searchquery.model.results.Bm25Parameters;
import nu.marginalia.api.searchquery.model.results.ResultRankingParameters;
import nu.marginalia.api.searchquery.model.results.debug.ResultRankingDetails;
import nu.marginalia.api.searchquery.model.results.debug.ResultRankingInputs;
import nu.marginalia.api.searchquery.model.results.debug.ResultRankingOutputs;
import nu.marginalia.index.query.limit.QueryLimits;
import nu.marginalia.index.query.limit.SpecificationLimit;
import nu.marginalia.index.query.limit.SpecificationLimitType;
@ -139,4 +142,45 @@ public class IndexProtobufCodec {
return builder.build();
}
public static RpcResultRankingDetails convertRankingDetails(ResultRankingDetails rankingDetails) {
if (rankingDetails == null) {
return null;
}
return RpcResultRankingDetails.newBuilder()
.setInputs(convertRankingInputs(rankingDetails.inputs()))
.setOutput(convertRankingOutput(rankingDetails.outputs()))
.build();
}
private static RpcResultRankingOutputs convertRankingOutput(ResultRankingOutputs outputs) {
return RpcResultRankingOutputs.newBuilder()
.setAverageSentenceLengthPenalty(outputs.averageSentenceLengthPenalty())
.setQualityPenalty(outputs.qualityPenalty())
.setRankingBonus(outputs.rankingBonus())
.setTopologyBonus(outputs.topologyBonus())
.setDocumentLengthPenalty(outputs.documentLengthPenalty())
.setTemporalBias(outputs.temporalBias())
.setFlagsPenalty(outputs.flagsPenalty())
.setOverallPart(outputs.overallPart())
.setTcfOverlap(outputs.tcfOverlap())
.setTcfJaccard(outputs.tcfJaccard())
.setBM25F(outputs.bM25F())
.setBM25N(outputs.bM25N())
.setBM25P(outputs.bM25P())
.build();
}
private static RpcResultRankingInputs convertRankingInputs(ResultRankingInputs inputs) {
return RpcResultRankingInputs.newBuilder()
.setRank(inputs.rank())
.setAsl(inputs.asl())
.setQuality(inputs.quality())
.setSize(inputs.size())
.setTopology(inputs.topology())
.setYear(inputs.year())
.addAllFlags(inputs.flags())
.build();
}
}

4
code/functions/search-query/api/java/nu/marginalia/api/searchquery/QueryProtobufCodec.java
View File

@ -171,9 +171,9 @@ public class QueryProtobufCodec {
inputs.getAsl(),
inputs.getQuality(),
inputs.getSize(),
inputs.getFlagsPenalty(),
inputs.getTopology(),
inputs.getYear()
inputs.getYear(),
inputs.getFlagsList()
);
}

4
code/functions/search-query/api/java/nu/marginalia/api/searchquery/model/results/debug/ResultRankingInputs.java
View File

@ -1,3 +1,5 @@
package nu.marginalia.api.searchquery.model.results.debug;
public record ResultRankingInputs(int rank, int asl, int quality, int size, int flagsPenalty, int topology, int year) {}
import java.util.List;
public record ResultRankingInputs(int rank, int asl, int quality, int size, int topology, int year, List<String> flags) {}

6
code/functions/search-query/api/src/main/protobuf/query-api.proto
View File

@ -150,9 +150,9 @@ message RpcResultRankingInputs {
int32 asl = 2;
int32 quality = 3;
int32 size = 4;
int32 flagsPenalty = 5;
int32 topology = 6;
int32 year = 7;
int32 topology = 5;
int32 year = 6;
repeated string flags = 7;
}
message RpcResultRankingOutputs {

43
code/index/java/nu/marginalia/index/IndexGrpcService.java
View File

@ -163,9 +163,8 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
.setBestPositions(result.bestPositions)
.setRawItem(rawItem);
var rankingDetails = convertRankingDetails(result.rankingDetails);
var rankingDetails = IndexProtobufCodec.convertRankingDetails(result.rankingDetails);
if (rankingDetails != null) {
logger.info(queryMarker, "Ranking details: {}", rankingDetails);
decoratedBuilder.setRankingDetails(rankingDetails);
}
@ -183,46 +182,6 @@ public class IndexGrpcService extends IndexApiGrpc.IndexApiImplBase {
}
}
private RpcResultRankingDetails convertRankingDetails(ResultRankingDetails rankingDetails) {
if (rankingDetails == null) {
return null;
}
return RpcResultRankingDetails.newBuilder()
.setInputs(convertRankingInputs(rankingDetails.inputs()))
.setOutput(convertRankingOutput(rankingDetails.outputs()))
.build();
}
private RpcResultRankingOutputs convertRankingOutput(ResultRankingOutputs outputs) {
return RpcResultRankingOutputs.newBuilder()
.setAverageSentenceLengthPenalty(outputs.averageSentenceLengthPenalty())
.setQualityPenalty(outputs.qualityPenalty())
.setRankingBonus(outputs.rankingBonus())
.setTopologyBonus(outputs.topologyBonus())
.setDocumentLengthPenalty(outputs.documentLengthPenalty())
.setTemporalBias(outputs.temporalBias())
.setFlagsPenalty(outputs.flagsPenalty())
.setOverallPart(outputs.overallPart())
.setTcfOverlap(outputs.tcfOverlap())
.setTcfJaccard(outputs.tcfJaccard())
.setBM25F(outputs.bM25F())
.setBM25N(outputs.bM25N())
.setBM25P(outputs.bM25P())
.build();
}
private RpcResultRankingInputs convertRankingInputs(ResultRankingInputs inputs) {
return RpcResultRankingInputs.newBuilder()
.setRank(inputs.rank())
.setAsl(inputs.asl())
.setQuality(inputs.quality())
.setSize(inputs.size())
.setFlagsPenalty(inputs.flagsPenalty())
.setTopology(inputs.topology())
.setYear(inputs.year())
.build();
}
// exists for test access
@SneakyThrows

14
code/index/java/nu/marginalia/index/results/IndexMetadataService.java
View File

@ -65,13 +65,15 @@ public class IndexMetadataService {
for (var term : searchQuery.searchTermsPriority) {
if (termToId.containsKey(term)) {
continue;
long id = SearchTermsUtil.getWordId(term);
termIdsPrio.add(id);
}
else {
long id = SearchTermsUtil.getWordId(term);
termIdsList.add(id);
termIdsPrio.add(id);
termToId.put(term, id);
}
long id = SearchTermsUtil.getWordId(term);
termIdsList.add(id);
termIdsPrio.add(id);
termToId.put(term, id);
}
return new QuerySearchTerms(termToId,

4
code/index/java/nu/marginalia/ranking/results/ResultValuator.java
View File

@ -99,9 +99,9 @@ public class ResultValuator {
asl,
quality,
size,
flagsPenalty,
topology,
year
year,
DocumentFlags.decode(documentMetadata).stream().map(Enum::name).toList()
),
new ResultRankingOutputs(
averageSentenceLengthPenalty,

26
code/services-core/query-service/resources/templates/qdebug.hdb
View File

@ -112,7 +112,31 @@
<p>{{description}}</p>
<div><small class="text-muted">dataHash: {{dataHash}} wordsTotal: {{wordsTotal}} bestPositions: {{bestPositions}} rankingScore: {{rankingScore}} urlQuality: {{urlQuality}}</small></div>
<div>{{rankingDetails}}</div>
{{#with rankingDetails.inputs}}
<div><small class="text-muted">Rank: {{rank}}</small></div>
<div><small class="text-muted">ASL: {{asl}}</small></div>
<div><small class="text-muted">Quality: {{quality}}</small></div>
<div><small class="text-muted">Size: {{size}}</small></div>
<div><small class="text-muted">Topology: {{topology}}</small></div>
<div><small class="text-muted">Year: {{year}}</small></div>
<div><small class="text-muted">Flags: {{#each flags}} {{.}} {{/each}}</small></div>
{{/with}}
{{#with rankingDetails.outputs}}
<div><small class="text-muted">Average Sentence Length Penalty: {{averageSentenceLengthPenalty}}</small></div>
<div><small class="text-muted">Quality Penalty: {{qualityPenalty}}</small></div>
<div><small class="text-muted">Ranking Bonus: {{rankingBonus}}</small></div>
<div><small class="text-muted">Topology Bonus: {{topologyBonus}}</small></div>
<div><small class="text-muted">Document Length Penalty: {{documentLengthPenalty}}</small></div>
<div><small class="text-muted">Temporal Bias: {{temporalBias}}</small></div>
<div><small class="text-muted">Flags Penalty: {{flagsPenalty}}</small></div>
<div><small class="text-muted">Overall Part: {{overallPart}}</small></div>
<div><small class="text-muted">TCF Overlap: {{tcfOverlap}}</small></div>
<div><small class="text-muted">TCF Jaccard: {{tcfJaccard}}</small></div>
<div><small class="text-muted">BM25 Full: {{bM25F}}</small></div>
<div><small class="text-muted">BM25 Ngram: {{bM25N}}</small></div>
<div><small class="text-muted">BM25 Prio: {{bM25P}}</small></div>
{{/with}}
</div>
{{/each}}
{{/if}}