Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(index journal; minor) Even cleaner separation of concerns.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-09-01 11:28:02 +02:00
parent 320dad7f1a
commit 10a74f45ea
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReader.java
View File

@ -19,10 +19,6 @@ public interface IndexJournalReader {
static IndexJournalReader paging(Path baseDir) throws IOException {
return new IndexJournalReaderPagingImpl(baseDir);
}
static IndexJournalReader filteringSingleFile(Path path, LongPredicate wordMetaFilter) throws IOException {
return new IndexJournalReaderSingleFile(path)
.filtering(wordMetaFilter);
}
default void forEachWordId(LongConsumer consumer) {
var ptr = this.newPointer();
@ -32,6 +28,7 @@ public interface IndexJournalReader {
}
}
}
default void forEachDocId(LongConsumer consumer) {
var ptr = this.newPointer();
while (ptr.nextDocument()) {
@ -46,10 +43,6 @@ public interface IndexJournalReader {
return new FilteringIndexJournalReader(this, termMetaFilter);
}
interface LongObjectConsumer<T> {
void accept(long left, T right);
}
}
class FilteringIndexJournalReader implements IndexJournalReader {

9
code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java
View File

@ -13,6 +13,7 @@ import nu.marginalia.index.journal.reader.IndexJournalReader;
import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.model.id.UrlIdCodec;
import nu.marginalia.model.idx.WordFlags;
import nu.marginalia.model.idx.WordMetadata;
import nu.marginalia.mq.MessageQueueFactory;
import nu.marginalia.mq.MqMessage;
import nu.marginalia.mq.inbox.MqInboxResponse;
@ -125,11 +126,15 @@ public class IndexConstructorMain {
Path tmpDir = indexStaging.asPath().resolve("tmp");
if (!Files.isDirectory(tmpDir)) Files.createDirectories(tmpDir);
// The priority index only includes words that have bits indicating they are
// important to the document. This filter will act on the encoded {@see WordMetadata}
LongPredicate wordMetaFilter = getPriorityIndexWordMetaFilter();
ReverseIndexConstructor.
createReverseIndex(heartbeat,
(path) -> IndexJournalReader.filteringSingleFile(path, wordMetaFilter),
(path) -> IndexJournalReader
.singleFile(path)
.filtering(wordMetaFilter),
indexStaging.asPath(), this::addRank, tmpDir, outputFileDocs, outputFileWords);
}
@ -145,7 +150,7 @@ public class IndexConstructorMain {
| WordFlags.Site.asBit()
| WordFlags.SiteAdjacent.asBit();
return r -> (r & highPriorityFlags) != 0;
return r -> WordMetadata.hasAnyFlags(r, highPriorityFlags);
}
private void createForwardIndex() throws SQLException, IOException {