Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixes

Browse Source
This commit is contained in:
Viktor Lofgren 2023-01-09 18:45:04 +01:00
parent 0fa1c8c16a
commit 11b0d61efc
3 changed files with 23 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java
View File

@ -129,13 +129,14 @@ public class IndexServicesFactory {
revIndexDoc.get(NEXT_PART).toPath());
converter.convert();
tryGc();
}
private void convertPriorityReverseIndex() throws IOException {
logger.info("Converting priority reverse index");
var longArray = LongArray.mmapRead(writerIndexFile.get(0).toPath());
var journalReader = new SearchIndexJournalReaderSingleFile(longArray, null, ReverseIndexPriorityParameters::filterPriorityRecord);
@ -146,6 +147,8 @@ public class IndexServicesFactory {
revPrioIndexDoc.get(NEXT_PART).toPath());
converter.convert();
tryGc();
}
private void convertForwardIndex() throws IOException {
@ -156,9 +159,23 @@ public class IndexServicesFactory {
fwdIndexDocId.get(NEXT_PART).toPath(),
fwdIndexDocData.get(NEXT_PART).toPath())
.convert();
tryGc();
}
public void tryGc() {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
System.runFinalization();
System.gc();
}
public ReverseIndexReader getReverseIndexReader() throws IOException {
return new ReverseIndexReader(
revIndexWords.get(LIVE_PART).toPath(),

4
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/forward/ForwardIndexConverter.java
View File

@ -1,5 +1,6 @@
package nu.marginalia.wmsa.edge.index.postings.forward;
import com.upserve.uppend.blobs.NativeIO;
import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import nu.marginalia.util.array.LongArray;
import nu.marginalia.wmsa.edge.index.postings.journal.reader.SearchIndexJournalReader;
@ -73,6 +74,9 @@ public class ForwardIndexConverter {
docFileData.force();
docsFileId.force();
docFileData.advice(NativeIO.Advice.DontNeed);
docsFileId.advice(NativeIO.Advice.DontNeed);
} catch (IOException ex) {
logger.error("Failed to convert", ex);
throw ex;

1
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java
View File

@ -75,6 +75,7 @@ public class ReverseIndexConverter {
StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE))
{
logger.info("Creating Intermediate Docs File");
// Construct intermediate index
try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE))
{