Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixes

Browse Source
This commit is contained in:
Viktor Lofgren 2023-01-09 22:06:15 +01:00
parent 4ec338d218
commit cb408dd737
1 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java
View File

@ -77,9 +77,11 @@ public class ReverseIndexConverter {
logger.info("Creating Intermediate Docs File"); logger.info("Creating Intermediate Docs File");
// Construct intermediate index // Construct intermediate index
try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE)) try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE);
IntermediateIndexConstructor intermediateIndexConstructor = new IntermediateIndexConstructor(tmpFileDir, wordsOffsets, intermediateDocumentWriteFunnel)
)
{ {
journalReader.forEachDocIdRecord(new IntermediateIndexConstructor(wordsOffsets, intermediateDocumentWriteFunnel)); journalReader.forEachDocIdRecord(intermediateIndexConstructor);
intermediateDocumentWriteFunnel.write(intermediateDocChannel); intermediateDocumentWriteFunnel.write(intermediateDocChannel);
} }
intermediateDocChannel.force(false); intermediateDocChannel.force(false);
@ -185,15 +187,19 @@ public class ReverseIndexConverter {
} }
} }
private static class IntermediateIndexConstructor implements SearchIndexJournalReaderSingleFile.LongObjectConsumer<SearchIndexJournalEntry.Record> { private static class IntermediateIndexConstructor implements SearchIndexJournalReaderSingleFile.LongObjectConsumer<SearchIndexJournalEntry.Record>, AutoCloseable {
private final LongArray wordRangeEnds; private final LongArray wordRangeEnds;
private final IntArray wordRangeOffset; private final IntArray wordRangeOffset;
private final RandomWriteFunnel documentsFile; private final RandomWriteFunnel documentsFile;
public IntermediateIndexConstructor(LongArray wordRangeEnds, RandomWriteFunnel documentsFile) { private final Path tempFile;
public IntermediateIndexConstructor(Path tempDir, LongArray wordRangeEnds, RandomWriteFunnel documentsFile) throws IOException {
tempFile = Files.createTempFile(tempDir, "iic", "dat");
this.wordRangeEnds = wordRangeEnds; this.wordRangeEnds = wordRangeEnds;
this.wordRangeOffset = IntArray.allocate(wordRangeEnds.size()); this.wordRangeOffset = IntArray.mmapForWriting(tempFile, wordRangeEnds.size());
this.documentsFile = documentsFile; this.documentsFile = documentsFile;
} }
@ -215,6 +221,9 @@ public class ReverseIndexConverter {
return wordRangeEnds.get(wordId - 1); return wordRangeEnds.get(wordId - 1);
} }
public void close() throws IOException {
Files.delete(tempFile);
}
} }
} }