diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java index 8ca580db..992e3ee5 100644 --- a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java +++ b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java @@ -9,7 +9,7 @@ import java.util.List; import java.util.regex.Pattern; public class QueryTokenizer { - private static final Pattern noisePattern = Pattern.compile("[,]"); + private static final Pattern noisePattern = Pattern.compile("[,\\s]"); public List tokenizeQuery(String rawQuery) { List tokens = new ArrayList<>(); diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java index d2a2ee6e..8bc19545 100644 --- a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java +++ b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java @@ -18,6 +18,29 @@ class QueryParserTest { parser = new QueryParser(); } + @Test + public void testTabHandling() { + var query = " lorem\tipsum\ndolor sit"; + var ret = parser.parse(query); + assertEquals(4, ret.size()); + + var lorem = ret.get(0); + assertEquals("lorem", lorem.str); + assertEquals("lorem", lorem.displayStr); + + var ipsum = ret.get(1); + assertEquals("ipsum", ipsum.str); + assertEquals("ipsum", ipsum.displayStr); + + var dolor = ret.get(2); + assertEquals("dolor", dolor.str); + assertEquals("dolor", dolor.displayStr); + + var sit = ret.get(3); + assertEquals("sit", sit.str); + assertEquals("sit", sit.displayStr); + } + @Test public void testAdviceString() { var ret = parser.parse("alcibiades (plato) \"my query\" -cars");