Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use a mapped file instead of allocating to save memory (#136)

Browse Source 
Co-authored-by: vlofgren <vlofgren@gmail.com>
Co-authored-by: vlofgren <vlofgren@marginalia.nu>
Co-authored-by: Viktor Lofgren <vlofgren@marginalia.nu>
Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/136
This commit is contained in:
Viktor Lofgren 2023-01-09 22:06:58 +01:00
parent a9ddc328a6
commit 4928b2e00e
1 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java
View File

@ -77,9 +77,11 @@ public class ReverseIndexConverter {
logger.info("Creating Intermediate Docs File");
// Construct intermediate index
try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE))
try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE);
IntermediateIndexConstructor intermediateIndexConstructor = new IntermediateIndexConstructor(tmpFileDir, wordsOffsets, intermediateDocumentWriteFunnel)
)
{
journalReader.forEachDocIdRecord(new IntermediateIndexConstructor(wordsOffsets, intermediateDocumentWriteFunnel));
journalReader.forEachDocIdRecord(intermediateIndexConstructor);
intermediateDocumentWriteFunnel.write(intermediateDocChannel);
}
intermediateDocChannel.force(false);
@ -185,15 +187,19 @@ public class ReverseIndexConverter {
}
}
private static class IntermediateIndexConstructor implements SearchIndexJournalReaderSingleFile.LongObjectConsumer<SearchIndexJournalEntry.Record> {
private static class IntermediateIndexConstructor implements SearchIndexJournalReaderSingleFile.LongObjectConsumer<SearchIndexJournalEntry.Record>, AutoCloseable {
private final LongArray wordRangeEnds;
private final IntArray wordRangeOffset;
private final RandomWriteFunnel documentsFile;
public IntermediateIndexConstructor(LongArray wordRangeEnds, RandomWriteFunnel documentsFile) {
private final Path tempFile;
public IntermediateIndexConstructor(Path tempDir, LongArray wordRangeEnds, RandomWriteFunnel documentsFile) throws IOException {
tempFile = Files.createTempFile(tempDir, "iic", "dat");
this.wordRangeEnds = wordRangeEnds;
this.wordRangeOffset = IntArray.allocate(wordRangeEnds.size());
this.wordRangeOffset = IntArray.mmapForWriting(tempFile, wordRangeEnds.size());
this.documentsFile = documentsFile;
}
@ -215,6 +221,9 @@ public class ReverseIndexConverter {
return wordRangeEnds.get(wordId - 1);
}
public void close() throws IOException {
Files.delete(tempFile);
}
}
}