Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(search) Tweak token formation.

Browse Source
This commit is contained in:
Viktor Lofgren 2025-01-05 21:01:09 +01:00
parent 94e1aa0baf
commit dcad0d7863
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code/libraries/language-processing/java/nu/marginalia/language/sentence/SentenceSegmentSplitter.java
View File

@ -93,10 +93,11 @@ public class SentenceSegmentSplitter {
if (part.startsWith("<") && part.endsWith(">") && part.length() > 2) { if (part.startsWith("<") && part.endsWith(">") && part.length() > 2) {
ret.set(i, part.substring(1, part.length() - 1)); ret.set(i, part.substring(1, part.length() - 1));
} }
else if (part.startsWith("'") && part.length() > 1) {
if (part.startsWith("'") && part.length() > 1) {
ret.set(i, part.substring(1)); ret.set(i, part.substring(1));
} }
else if (part.endsWith("'") && part.length() > 1) { if (part.endsWith("'") && part.length() > 1) {
ret.set(i, part.substring(0, part.length()-1)); ret.set(i, part.substring(0, part.length()-1));
} }