Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixes

Browse Source
This commit is contained in:
Viktor Lofgren 2023-01-09 18:45:04 +01:00
parent 0fa1c8c16a
commit 11b0d61efc
3 changed files with 23 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java
View File

@ -129,13 +129,14 @@ public class IndexServicesFactory {
revIndexDoc.get(NEXT_PART).toPath()); revIndexDoc.get(NEXT_PART).toPath());
converter.convert(); converter.convert();
tryGc();
} }
private void convertPriorityReverseIndex() throws IOException { private void convertPriorityReverseIndex() throws IOException {
logger.info("Converting priority reverse index"); logger.info("Converting priority reverse index");
var longArray = LongArray.mmapRead(writerIndexFile.get(0).toPath()); var longArray = LongArray.mmapRead(writerIndexFile.get(0).toPath());
var journalReader = new SearchIndexJournalReaderSingleFile(longArray, null, ReverseIndexPriorityParameters::filterPriorityRecord); var journalReader = new SearchIndexJournalReaderSingleFile(longArray, null, ReverseIndexPriorityParameters::filterPriorityRecord);
@ -146,6 +147,8 @@ public class IndexServicesFactory {
revPrioIndexDoc.get(NEXT_PART).toPath()); revPrioIndexDoc.get(NEXT_PART).toPath());
converter.convert(); converter.convert();
tryGc();
} }
private void convertForwardIndex() throws IOException { private void convertForwardIndex() throws IOException {
@ -156,9 +159,23 @@ public class IndexServicesFactory {
fwdIndexDocId.get(NEXT_PART).toPath(), fwdIndexDocId.get(NEXT_PART).toPath(),
fwdIndexDocData.get(NEXT_PART).toPath()) fwdIndexDocData.get(NEXT_PART).toPath())
.convert(); .convert();
tryGc();
} }
public void tryGc() {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
System.runFinalization();
System.gc();
}
public ReverseIndexReader getReverseIndexReader() throws IOException { public ReverseIndexReader getReverseIndexReader() throws IOException {
return new ReverseIndexReader( return new ReverseIndexReader(
revIndexWords.get(LIVE_PART).toPath(), revIndexWords.get(LIVE_PART).toPath(),

4
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/forward/ForwardIndexConverter.java
View File

@ -1,5 +1,6 @@
package nu.marginalia.wmsa.edge.index.postings.forward; package nu.marginalia.wmsa.edge.index.postings.forward;
import com.upserve.uppend.blobs.NativeIO;
import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import nu.marginalia.util.array.LongArray; import nu.marginalia.util.array.LongArray;
import nu.marginalia.wmsa.edge.index.postings.journal.reader.SearchIndexJournalReader; import nu.marginalia.wmsa.edge.index.postings.journal.reader.SearchIndexJournalReader;
@ -73,6 +74,9 @@ public class ForwardIndexConverter {
docFileData.force(); docFileData.force();
docsFileId.force(); docsFileId.force();
docFileData.advice(NativeIO.Advice.DontNeed);
docsFileId.advice(NativeIO.Advice.DontNeed);
} catch (IOException ex) { } catch (IOException ex) {
logger.error("Failed to convert", ex); logger.error("Failed to convert", ex);
throw ex; throw ex;

1
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java
View File

@ -75,6 +75,7 @@ public class ReverseIndexConverter {
StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE)) StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE))
{ {
logger.info("Creating Intermediate Docs File"); logger.info("Creating Intermediate Docs File");
// Construct intermediate index // Construct intermediate index
try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE)) try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE))
{ {