Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 13:19:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(index journal; minor) Even cleaner separation of concerns.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-09-01 11:28:02 +02:00
parent 320dad7f1a
commit 10a74f45ea
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReader.java
View File

@ -19,10 +19,6 @@ public interface IndexJournalReader {
static IndexJournalReader paging(Path baseDir) throws IOException { static IndexJournalReader paging(Path baseDir) throws IOException {
return new IndexJournalReaderPagingImpl(baseDir); return new IndexJournalReaderPagingImpl(baseDir);
} }
static IndexJournalReader filteringSingleFile(Path path, LongPredicate wordMetaFilter) throws IOException {
return new IndexJournalReaderSingleFile(path)
.filtering(wordMetaFilter);
}
default void forEachWordId(LongConsumer consumer) { default void forEachWordId(LongConsumer consumer) {
var ptr = this.newPointer(); var ptr = this.newPointer();
@ -32,6 +28,7 @@ public interface IndexJournalReader {
} }
} }
} }
default void forEachDocId(LongConsumer consumer) { default void forEachDocId(LongConsumer consumer) {
var ptr = this.newPointer(); var ptr = this.newPointer();
while (ptr.nextDocument()) { while (ptr.nextDocument()) {
@ -46,10 +43,6 @@ public interface IndexJournalReader {
return new FilteringIndexJournalReader(this, termMetaFilter); return new FilteringIndexJournalReader(this, termMetaFilter);
} }
interface LongObjectConsumer<T> {
void accept(long left, T right);
}
} }
class FilteringIndexJournalReader implements IndexJournalReader { class FilteringIndexJournalReader implements IndexJournalReader {

9
code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java
View File

@ -13,6 +13,7 @@ import nu.marginalia.index.journal.reader.IndexJournalReader;
import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.model.id.UrlIdCodec;
import nu.marginalia.model.idx.WordFlags; import nu.marginalia.model.idx.WordFlags;
import nu.marginalia.model.idx.WordMetadata;
import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.mq.MessageQueueFactory;
import nu.marginalia.mq.MqMessage; import nu.marginalia.mq.MqMessage;
import nu.marginalia.mq.inbox.MqInboxResponse; import nu.marginalia.mq.inbox.MqInboxResponse;
@ -125,11 +126,15 @@ public class IndexConstructorMain {
Path tmpDir = indexStaging.asPath().resolve("tmp"); Path tmpDir = indexStaging.asPath().resolve("tmp");
if (!Files.isDirectory(tmpDir)) Files.createDirectories(tmpDir); if (!Files.isDirectory(tmpDir)) Files.createDirectories(tmpDir);
// The priority index only includes words that have bits indicating they are
// important to the document. This filter will act on the encoded {@see WordMetadata}
LongPredicate wordMetaFilter = getPriorityIndexWordMetaFilter(); LongPredicate wordMetaFilter = getPriorityIndexWordMetaFilter();
ReverseIndexConstructor. ReverseIndexConstructor.
createReverseIndex(heartbeat, createReverseIndex(heartbeat,
(path) -> IndexJournalReader.filteringSingleFile(path, wordMetaFilter), (path) -> IndexJournalReader
.singleFile(path)
.filtering(wordMetaFilter),
indexStaging.asPath(), this::addRank, tmpDir, outputFileDocs, outputFileWords); indexStaging.asPath(), this::addRank, tmpDir, outputFileDocs, outputFileWords);
} }
@ -145,7 +150,7 @@ public class IndexConstructorMain {
| WordFlags.Site.asBit() | WordFlags.Site.asBit()
| WordFlags.SiteAdjacent.asBit(); | WordFlags.SiteAdjacent.asBit();
return r -> (r & highPriorityFlags) != 0; return r -> WordMetadata.hasAnyFlags(r, highPriorityFlags);
} }
private void createForwardIndex() throws SQLException, IOException { private void createForwardIndex() throws SQLException, IOException {