Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(search) Adjust token formation rules to be more lenient to C++ and PHP code.

Browse Source 
This addresses Issue #142
This commit is contained in:
Viktor Lofgren 2025-01-05 20:50:27 +01:00
parent 9b2ceaf37c
commit b62f043910
2 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceSegmentSplitter.java
View File

@ -27,7 +27,7 @@ public class SentenceSegmentSplitter {
else {
// If we flatten unicode, we do this...
// FIXME: This can almost definitely be cleaned up and simplified.
wordBreakPattern = Pattern.compile("([^/_#@.a-zA-Z'+\\-0-9\\u00C0-\\u00D6\\u00D8-\\u00f6\\u00f8-\\u00ff]+)|[|]|(\\.(\\s+|$))");
wordBreakPattern = Pattern.compile("([^/<>$:_#@.a-zA-Z'+\\-0-9\\u00C0-\\u00D6\\u00D8-\\u00f6\\u00f8-\\u00ff]+)|[|]|(\\.(\\s+|$))");
}
}

14
code/libraries/language-processing/test/nu/marginalia/language/sentence/SentenceExtractorTest.java
View File

@ -28,6 +28,20 @@ class SentenceExtractorTest {
System.out.println(dld);
}
@Test
void testCplusplus() {
var dld = sentenceExtractor.extractSentence("std::vector", EnumSet.noneOf(HtmlTag.class));
assertEquals(1, dld.length());
assertEquals("std::vector", dld.wordsLowerCase[0]);
}
@Test
void testPHP() {
var dld = sentenceExtractor.extractSentence("$_GET", EnumSet.noneOf(HtmlTag.class));
assertEquals(1, dld.length());
assertEquals("$_get", dld.wordsLowerCase[0]);
}
@Test
void testPolishArtist() {
var dld = sentenceExtractor.extractSentence("Uklański", EnumSet.noneOf(HtmlTag.class));