Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixing a bug where search terms would sometimes be ignored, tweaking timeouts, adding debug feature for the search service.

Browse Source
This commit is contained in:
vlofgren 2022-09-12 21:08:53 +02:00
parent 73a5906c12
commit 297f8e4cd7
6 changed files with 15 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
marginalia_nu/src/main/java/nu/marginalia/util/dict/DictionaryData.java
View File

@ -12,7 +12,7 @@ public class DictionaryData {
private final int DICTIONARY_BANK_SIZE; private final int DICTIONARY_BANK_SIZE;
private static final Logger logger = LoggerFactory.getLogger(DictionaryData.class); private static final Logger logger = LoggerFactory.getLogger(DictionaryData.class);
private final ArrayList<DictionaryDataBank> banks = new ArrayList(100); private final ArrayList<DictionaryDataBank> banks = new ArrayList<>(100);
public DictionaryData(int bankSize) { public DictionaryData(int bankSize) {
DICTIONARY_BANK_SIZE = bankSize; DICTIONARY_BANK_SIZE = bankSize;

4
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeIndexBucket.java
View File

@ -136,10 +136,6 @@ public class EdgeIndexBucket {
query = query.not(term); query = query.not(term);
} }
for (int term : orderedIncludes) {
query.prioritize(term);
}
return query.build(); return query.build();
} }

4
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/svc/EdgeIndexQueryService.java
View File

@ -41,7 +41,7 @@ public class EdgeIndexQueryService {
private final Logger logger = LoggerFactory.getLogger(getClass()); private final Logger logger = LoggerFactory.getLogger(getClass());
private static final int SEARCH_BUDGET_TIMEOUT_MS = 3000; private static final int SEARCH_BUDGET_TIMEOUT_MS = 250;
private static final int QUERY_FETCH_SIZE = 8192; private static final int QUERY_FETCH_SIZE = 8192;
private static final int QUERY_FIRST_PASS_DOMAIN_LIMIT = 64; private static final int QUERY_FIRST_PASS_DOMAIN_LIMIT = 64;
@ -301,7 +301,7 @@ public class EdgeIndexQueryService {
var word = lookUpWord(include); var word = lookUpWord(include);
if (word.isEmpty()) { if (word.isEmpty()) {
logger.debug("Unknown search term: " + include); logger.debug("Unknown search term: " + include);
return new EdgeIndexSearchTerms(includes, excludes); return new EdgeIndexSearchTerms(Collections.emptyList(), Collections.emptyList());
} }
includes.add(word.getAsInt()); includes.add(word.getAsInt());
} }

28
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/svc/query/IndexQuery.java
View File

@ -11,7 +11,6 @@ import static java.lang.Math.min;
public class IndexQuery { public class IndexQuery {
private final List<EntrySource> sources; private final List<EntrySource> sources;
private final List<QueryFilterStepIf> inclusionFilter = new ArrayList<>(10); private final List<QueryFilterStepIf> inclusionFilter = new ArrayList<>(10);
private final List<QueryFilterStepIf> priorityFilter = new ArrayList<>(10);
public IndexQuery(List<EntrySource> sources) { public IndexQuery(List<EntrySource> sources) {
this.sources = sources; this.sources = sources;
@ -21,10 +20,6 @@ public class IndexQuery {
inclusionFilter.add(filter); inclusionFilter.add(filter);
} }
public void addPriorityFilter(QueryFilterStepIf filter) {
priorityFilter.add(filter);
}
private int si = 0; private int si = 0;
public boolean hasMore() { public boolean hasMore() {
@ -50,34 +45,11 @@ public class IndexQuery {
} }
} }
if (budget.hasTimeLeft()) {
prioritizeBuffer(dest, source, bufferUtilizedLength, budget);
}
int count = min(bufferUtilizedLength, dest.length); int count = min(bufferUtilizedLength, dest.length);
System.arraycopy(dest, 0, dest, 0, count); System.arraycopy(dest, 0, dest, 0, count);
return count; return count;
} }
private void prioritizeBuffer(long[] dest, EntrySource source, int remainingBufferSize, IndexSearchBudget budget) {
int prioStart = 0;
for (var filter : priorityFilter) {
if (!budget.hasTimeLeft())
break;
if (filter.getIndex() == source.getIndex())
continue;
prioStart += filter.retainReorder(dest, prioStart, remainingBufferSize);
if (prioStart >= remainingBufferSize) {
break;
}
}
}
public String toString() { public String toString() {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("Sources:\n"); sb.append("Sources:\n");

9
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/svc/query/IndexQueryFactory.java
View File

@ -84,15 +84,6 @@ public class IndexQueryFactory {
return this; return this;
} }
public void prioritize(int termId) {
for (var idx : priortyIndices) {
var range = idx.rangeForWord(cachePool, termId);
if (range.isPresent()) {
query.addPriorityFilter(new QueryFilterBTreeRange(idx, range, cachePool));
}
}
}
public IndexQuery build() { public IndexQuery build() {
return query; return query;
} }

21
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/results/SearchResultDecorator.java
View File

@ -20,6 +20,8 @@ public class SearchResultDecorator {
private final SearchResultValuator valuator; private final SearchResultValuator valuator;
private final Logger logger = LoggerFactory.getLogger(getClass()); private final Logger logger = LoggerFactory.getLogger(getClass());
private final boolean dumpTermData = Boolean.getBoolean("search-dump-term-data");
@Inject @Inject
public SearchResultDecorator(EdgeDataStoreDao edgeDataStoreDao, SearchResultValuator valuator) { public SearchResultDecorator(EdgeDataStoreDao edgeDataStoreDao, SearchResultValuator valuator) {
this.edgeDataStoreDao = edgeDataStoreDao; this.edgeDataStoreDao = edgeDataStoreDao;
@ -76,15 +78,16 @@ public class SearchResultDecorator {
final double value = valuator.evaluateTerms(resultItem.scores, details.words, details.title.length()); final double value = valuator.evaluateTerms(resultItem.scores, details.words, details.title.length());
// System.out.println("---"); if (dumpTermData) {
// System.out.println(details.getUrl()); System.out.println("---");
// System.out.println(details.getTitle()); System.out.println(details.getUrl());
// System.out.println(details.words); System.out.println(details.getTitle());
// for (var score : resultItem.scores) { System.out.println(details.words);
// System.out.println(score); for (var score : resultItem.scores) {
// } System.out.println(score);
// System.out.println(value); }
System.out.println(value);
}
return value + statePenalty; return value + statePenalty;
} }