From a9ddc328a6936aa38a69616fbd4b0461102a0e5b Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Mon, 9 Jan 2023 18:47:04 +0100 Subject: [PATCH] Fixes from master (#135) Co-authored-by: vlofgren Co-authored-by: vlofgren Co-authored-by: Viktor Lofgren Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/135 --- .../wmsa/edge/index/IndexServicesFactory.java | 19 ++++++++++++++++++- .../forward/ForwardIndexConverter.java | 4 ++++ .../reverse/ReverseIndexConverter.java | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java index 199418de..d069ec05 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/IndexServicesFactory.java @@ -129,13 +129,14 @@ public class IndexServicesFactory { revIndexDoc.get(NEXT_PART).toPath()); converter.convert(); + + tryGc(); } private void convertPriorityReverseIndex() throws IOException { logger.info("Converting priority reverse index"); - var longArray = LongArray.mmapRead(writerIndexFile.get(0).toPath()); var journalReader = new SearchIndexJournalReaderSingleFile(longArray, null, ReverseIndexPriorityParameters::filterPriorityRecord); @@ -146,6 +147,8 @@ public class IndexServicesFactory { revPrioIndexDoc.get(NEXT_PART).toPath()); converter.convert(); + + tryGc(); } private void convertForwardIndex() throws IOException { @@ -156,9 +159,23 @@ public class IndexServicesFactory { fwdIndexDocId.get(NEXT_PART).toPath(), fwdIndexDocData.get(NEXT_PART).toPath()) .convert(); + + tryGc(); } + public void tryGc() { + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + System.runFinalization(); + System.gc(); + } + public ReverseIndexReader getReverseIndexReader() throws IOException { return new ReverseIndexReader( revIndexWords.get(LIVE_PART).toPath(), diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/forward/ForwardIndexConverter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/forward/ForwardIndexConverter.java index c4113440..e066f734 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/forward/ForwardIndexConverter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/forward/ForwardIndexConverter.java @@ -1,5 +1,6 @@ package nu.marginalia.wmsa.edge.index.postings.forward; +import com.upserve.uppend.blobs.NativeIO; import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; import nu.marginalia.util.array.LongArray; import nu.marginalia.wmsa.edge.index.postings.journal.reader.SearchIndexJournalReader; @@ -73,6 +74,9 @@ public class ForwardIndexConverter { docFileData.force(); docsFileId.force(); + + docFileData.advice(NativeIO.Advice.DontNeed); + docsFileId.advice(NativeIO.Advice.DontNeed); } catch (IOException ex) { logger.error("Failed to convert", ex); throw ex; diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java index bd32dd68..32211b3c 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java @@ -75,6 +75,7 @@ public class ReverseIndexConverter { StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE)) { logger.info("Creating Intermediate Docs File"); + // Construct intermediate index try (RandomWriteFunnel intermediateDocumentWriteFunnel = new RandomWriteFunnel(tmpFileDir, RWF_BIN_SIZE)) {