Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixing a bug where search terms would sometimes be ignored, tweaking timeouts, adding debug feature for the search service.

Browse Source
This commit is contained in:
vlofgren 2022-09-12 21:08:53 +02:00
parent 73a5906c12
commit 297f8e4cd7
6 changed files with 15 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
marginalia_nu/src/main/java/nu/marginalia/util/dict/DictionaryData.java
View File

@ -12,7 +12,7 @@ public class DictionaryData {
private final int DICTIONARY_BANK_SIZE;
private static final Logger logger = LoggerFactory.getLogger(DictionaryData.class);
private final ArrayList<DictionaryDataBank> banks = new ArrayList(100);
private final ArrayList<DictionaryDataBank> banks = new ArrayList<>(100);
public DictionaryData(int bankSize) {
DICTIONARY_BANK_SIZE = bankSize;

4
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexBucket.java
View File

@ -136,10 +136,6 @@ public class EdgeIndexBucket {
query = query.not(term);
}
for (int term : orderedIncludes) {
query.prioritize(term);
}
return query.build();
}

4
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/svc/EdgeIndexQueryService.java
View File

@ -41,7 +41,7 @@ public class EdgeIndexQueryService {
private final Logger logger = LoggerFactory.getLogger(getClass());
private static final int SEARCH_BUDGET_TIMEOUT_MS = 3000;
private static final int SEARCH_BUDGET_TIMEOUT_MS = 250;
private static final int QUERY_FETCH_SIZE = 8192;
private static final int QUERY_FIRST_PASS_DOMAIN_LIMIT = 64;
@ -301,7 +301,7 @@ public class EdgeIndexQueryService {
var word = lookUpWord(include);
if (word.isEmpty()) {
logger.debug("Unknown search term: " + include);
return new EdgeIndexSearchTerms(includes, excludes);
return new EdgeIndexSearchTerms(Collections.emptyList(), Collections.emptyList());
}
includes.add(word.getAsInt());
}

28
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/svc/query/IndexQuery.java
View File

@ -11,7 +11,6 @@ import static java.lang.Math.min;
public class IndexQuery {
private final List<EntrySource> sources;
private final List<QueryFilterStepIf> inclusionFilter = new ArrayList<>(10);
private final List<QueryFilterStepIf> priorityFilter = new ArrayList<>(10);
public IndexQuery(List<EntrySource> sources) {
this.sources = sources;
@ -21,10 +20,6 @@ public class IndexQuery {
inclusionFilter.add(filter);
}
public void addPriorityFilter(QueryFilterStepIf filter) {
priorityFilter.add(filter);
}
private int si = 0;
public boolean hasMore() {
@ -50,34 +45,11 @@ public class IndexQuery {
}
}
if (budget.hasTimeLeft()) {
prioritizeBuffer(dest, source, bufferUtilizedLength, budget);
}
int count = min(bufferUtilizedLength, dest.length);
System.arraycopy(dest, 0, dest, 0, count);
return count;
}
private void prioritizeBuffer(long[] dest, EntrySource source, int remainingBufferSize, IndexSearchBudget budget) {
int prioStart = 0;
for (var filter : priorityFilter) {
if (!budget.hasTimeLeft())
break;
if (filter.getIndex() == source.getIndex())
continue;
prioStart += filter.retainReorder(dest, prioStart, remainingBufferSize);
if (prioStart >= remainingBufferSize) {
break;
}
}
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("Sources:\n");

9
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/svc/query/IndexQueryFactory.java
View File

@ -84,15 +84,6 @@ public class IndexQueryFactory {
return this;
}
public void prioritize(int termId) {
for (var idx : priortyIndices) {
var range = idx.rangeForWord(cachePool, termId);
if (range.isPresent()) {
query.addPriorityFilter(new QueryFilterBTreeRange(idx, range, cachePool));
}
}
}
public IndexQuery build() {
return query;
}

21
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/results/SearchResultDecorator.java
View File

@ -20,6 +20,8 @@ public class SearchResultDecorator {
private final SearchResultValuator valuator;
private final Logger logger = LoggerFactory.getLogger(getClass());
private final boolean dumpTermData = Boolean.getBoolean("search-dump-term-data");
@Inject
public SearchResultDecorator(EdgeDataStoreDao edgeDataStoreDao, SearchResultValuator valuator) {
this.edgeDataStoreDao = edgeDataStoreDao;
@ -76,15 +78,16 @@ public class SearchResultDecorator {
final double value = valuator.evaluateTerms(resultItem.scores, details.words, details.title.length());
// System.out.println("---");
// System.out.println(details.getUrl());
// System.out.println(details.getTitle());
// System.out.println(details.words);
// for (var score : resultItem.scores) {
// System.out.println(score);
// }
// System.out.println(value);
if (dumpTermData) {
System.out.println("---");
System.out.println(details.getUrl());
System.out.println(details.getTitle());
System.out.println(details.words);
for (var score : resultItem.scores) {
System.out.println(score);
}
System.out.println(value);
}
return value + statePenalty;
}