Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Clean up preconverter code

Browse Source
This commit is contained in:
vlofgren 2022-08-08 18:08:18 +02:00
parent 2af2c50f34
commit 0f59675f7c
8 changed files with 67 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchProfile.java
View File

@ -1,6 +1,8 @@
package nu.marginalia.wmsa.edge.search;
import nu.marginalia.wmsa.edge.converting.processor.logic.HtmlFeature;
import nu.marginalia.wmsa.edge.index.model.IndexBlock;
import nu.marginalia.wmsa.edge.model.search.EdgeSearchSubquery;
import java.util.Arrays;
import java.util.List;
@ -25,6 +27,9 @@ public enum EdgeSearchProfile {
ACADEMIA("academia",
List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords),
3),
FOOD("food",
List.of(IndexBlock.TitleKeywords, IndexBlock.Title, IndexBlock.Top, IndexBlock.Middle, IndexBlock.Low, IndexBlock.Link, IndexBlock.Words, IndexBlock.NamesWords),
2, 0),
;
@ -49,7 +54,15 @@ public enum EdgeSearchProfile {
case "default" -> DEFAULT;
case "corpo" -> CORPO;
case "academia" -> ACADEMIA;
case "food" -> FOOD;
default -> YOLO;
};
}
public void addTacitTerms(EdgeSearchSubquery subquery) {
if (this == FOOD) {
subquery.searchTermsInclude.add(HtmlFeature.CATEGORY_FOOD.getKeyword());
}
}
}

7
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/command/SearchJsParameter.java
View File

@ -1,6 +1,9 @@
package nu.marginalia.wmsa.edge.search.command;
import nu.marginalia.wmsa.edge.model.search.EdgeSearchSubquery;
import javax.annotation.Nullable;
import java.util.Arrays;
public enum SearchJsParameter {
DEFAULT("default"),
@ -21,4 +24,8 @@ public enum SearchJsParameter {
return DEFAULT;
}
public void addTacitTerms(EdgeSearchSubquery subquery) {
subquery.searchTermsExclude.addAll(Arrays.asList(implictExcludeSearchTerms));
}
}

8
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/EnglishDictionary.java
View File

@ -46,9 +46,15 @@ public class EnglishDictionary {
var variants = findWordVariants(s);
long freqBaseline = dict.getTermFreq(s);
return variants.stream()
var ret = variants.stream()
.filter(var -> freqBaseline*10 > dict.getTermFreq(var) && freqBaseline/10 < dict.getTermFreq(var)
).collect(Collectors.toList());
if (s.equals("recipe") || s.equals("recipes")) {
ret.add("category:food");
}
return ret;
}

12
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryFactory.java
View File

@ -54,7 +54,6 @@ public class QueryFactory {
}
private List<EdgeSearchSubquery> reevaluateSubqueries(EdgeSearchQuery processedQuery, EdgeUserSearchParameters params) {
final var jsSetting = params.jsSetting();
final var profile = params.profile();
List<EdgeSearchSubquery> subqueries =
@ -66,10 +65,6 @@ public class QueryFactory {
}
}
subqueries.forEach(sq -> {
sq.searchTermsExclude.addAll(Arrays.asList(jsSetting.implictExcludeSearchTerms));
});
subqueries.sort(Comparator.comparing(sq -> -sq.termSize()*2.3 + sq.block.sortOrder));
return subqueries;
@ -132,7 +127,12 @@ public class QueryFactory {
}
}
subqueries.add(new EdgeSearchSubquery(searchTermsInclude, searchTermsExclude, IndexBlock.TitleKeywords));
EdgeSearchSubquery subquery = new EdgeSearchSubquery(searchTermsInclude, searchTermsExclude, IndexBlock.TitleKeywords);
params.profile().addTacitTerms(subquery);
params.jsSetting().addTacitTerms(subquery);
subqueries.add(subquery);
}

6
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/query/QueryVariants.java
View File

@ -5,12 +5,12 @@ import com.google.inject.Singleton;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.ToString;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.KeywordExtractor;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.util.language.processing.model.DocumentSentence;
import nu.marginalia.util.language.processing.model.WordSpan;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import opennlp.tools.stemmer.PorterStemmer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -115,7 +115,6 @@ public class QueryVariants {
for (var span : goodSpans) {
alternativeQueries.addAll(joinTerms(span));
// alternativeQueries.addAll(swapTerms(span));
}
for (var ls : goodSpans) {
@ -134,8 +133,11 @@ public class QueryVariants {
}
}
QueryVariantSet returnValue = new QueryVariantSet();
returnValue.faithful.addAll(evaluateQueries(faithfulQueries));
returnValue.faithful.addAll(evaluateQueries(alternativeQueries));
returnValue.faithful.sort(Comparator.comparing(QueryVariant::getValue));

1
marginalia_nu/src/main/resources/static/edge/index.html
View File

@ -38,6 +38,7 @@
<option value="modern">Blogs and Personal Websites</option>
<option value="academia">Academia, Forums, Big Websites</option>
<option value="yolo" selected>Default Ranking Algorithm</option>
<option value="food">Recipes &#127859;</option>
<option value="corpo">Experimental</option>
</select>
<select name="js" id="js">

1
marginalia_nu/src/main/resources/templates/edge/parts/search-form.hdb
View File

@ -11,6 +11,7 @@
<option {{#eq profile "modern"}}selected{{/eq}} value="modern">Blogs and Personal Websites</option>
<option {{#eq profile "academia"}}selected{{/eq}} value="academia">Academia, Forums, Big Websites</option>
<option {{#eq profile "yolo"}}selected{{/eq}} value="yolo">Default Ranking Algorithm</option>
<option {{#eq profile "food"}}selected{{/eq}} value="food">Recipes &#127859;</option>
<option {{#eq profile "corpo"}}selected{{/eq}} value="corpo">Experimental</option>
</select>
<select name="js" id="js">

52
marginalia_nu/src/test/java/nu/marginalia/wmsa/edge/search/query/QueryVariantsTest.java
View File

@ -1,14 +1,12 @@
package nu.marginalia.wmsa.edge.search.query;
import nu.marginalia.util.TestLanguageModels;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.util.language.processing.SentenceExtractor;
import nu.marginalia.wmsa.edge.assistant.dict.NGramDict;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
class QueryVariantsTest {
QueryVariants variants;
QueryParser parser;
@ -25,30 +23,36 @@ class QueryVariantsTest {
parser = new QueryParser(new EnglishDictionary(dict), variants);
}
@Test @SuppressWarnings("unchecked")
void getQueryVariants() {
@Test
void getQueryFood() {
System.out.println(se.extractSentence("we are alone"));
testCase("DOS", List.of("DOS"));
testCase("dos", List.of("dos"));
testCase("we are alone", List.of("dos"));
testCase("3D Realms", List.of("dos"));
testCase("I am alone", List.of("dos"));
testCase("plato cave", List.of("dos"));
testCase("The internet is dead", List.of("dos"));
testCase("TRS80", List.of("trs_80"), List.of("trs80"));
testCase("TRS-80", List.of("trs-80"), List.of("trs80"));
testCase("TRS-80", List.of("trs-80"), List.of("trs80"));
testCase("Raspberry Pi 2", List.of("trs-80"), List.of("trs80"));
testCase("Duke Nukem 3D", List.of("trs-80"), List.of("trs80"));
testCase("The Man of Tomorrow", List.of("trs-80"), List.of("trs80"));
testCase("Computer Manual", List.of("trs-80"), List.of("trs80"));
testCase("Knitting", List.of("trs-80"), List.of("trs80"));
testCase("capcom", List.of("trs-80"), List.of("trs80"));
testCase("the man of tomorrow", List.of("trs-80"), List.of("trs80"));
testCase("Omelet recipe");
}
private void testCase(String input, List<String>... expected) {
@Test
void getQueryVariants() {
System.out.println(se.extractSentence("we are alone"));
testCase("DOS");
testCase("dos");
testCase("we are alone");
testCase("3D Realms");
testCase("I am alone");
testCase("plato cave");
testCase("The internet is dead");
testCase("TRS80");
testCase("TRS-80");
testCase("TRS-80");
testCase("Raspberry Pi 2");
testCase("Duke Nukem 3D");
testCase("The Man of Tomorrow");
testCase("Computer Manual");
testCase("Knitting");
testCase("capcom");
testCase("the man of tomorrow");
}
private void testCase(String input) {
var tokens = variants.getQueryVariants(parser.extractBasicTokens(input));
System.out.println(tokens);
}