Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bitmask calculation improvement. Take sentence length into consideration, not all lines are equal.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-03-30 15:42:06 +02:00
parent 16e37672fc
commit 137adb9c3c
1 changed files with 30 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/extractors/KeywordPositionBitmask.java
View File

@ -39,7 +39,7 @@ public class KeywordPositionBitmask {
positionMask.merge(sent.constructStemmedWordFromSpan(span), posBit, this::bitwiseOr); positionMask.merge(sent.constructStemmedWordFromSpan(span), posBit, this::bitwiseOr);
} }
linePos.next(); linePos.next(sent.length());
} }
} }
@ -52,33 +52,49 @@ public class KeywordPositionBitmask {
} }
private static class LinePosition { private static class LinePosition {
private int lineLengthCtr = 0;
private int line = 0; private int line = 0;
private int pos = 1; private int bitMaskPos = 1;
public int pos() { public int pos() {
return pos; return bitMaskPos;
} }
public void next() { public void next(int sentenceLength) {
if (pos < 4) pos ++; if (bitMaskPos < 4) bitMaskPos++;
else if (pos < 8) { else if (bitMaskPos < 8) {
if (++line >= 2) { if (advanceLine(sentenceLength)>= 2) {
pos++; bitMaskPos++;
line = 0; line = 0;
} }
} }
else if (pos < 24) { else if (bitMaskPos < 24) {
if (++line >= 4) { if (advanceLine(sentenceLength) >= 4) {
pos++; bitMaskPos++;
line = 0; line = 0;
} }
} }
else if (pos < 64) { else if (bitMaskPos < 64) {
if (++line > 8) { if (advanceLine(sentenceLength) > 8) {
pos++; bitMaskPos++;
line = 0; line = 0;
} }
} }
} }
private int advanceLine(int sentenceLength) {
if (sentenceLength > 10) {
lineLengthCtr = 0;
return ++line;
}
lineLengthCtr += sentenceLength;
if (lineLengthCtr > 15) {
lineLengthCtr = 0;
return ++line;
}
return line;
}
} }
} }