From b0a874a84247b57e94526bcf54aa101fd48dca03 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 18 Aug 2024 11:05:27 +0200 Subject: [PATCH] (*) Upgrade slop library -> 0.0.5 --- .../construction/ForwardIndexConverter.java | 4 +- .../index/journal/IndexJournalPage.java | 18 ++-- .../index/journal/IndexJournalSlopWriter.java | 20 ++--- .../full/FullPreindexDocuments.java | 12 +-- .../full/FullPreindexWordSegments.java | 6 +- .../prio/PrioPreindexDocuments.java | 10 +-- .../prio/PrioPreindexWordSegments.java | 8 +- .../model/processed/SlopDocumentRecord.java | 86 +++++++++---------- .../model/processed/SlopDomainLinkRecord.java | 12 +-- .../model/processed/SlopDomainRecord.java | 46 +++++----- settings.gradle | 2 +- 11 files changed, 112 insertions(+), 112 deletions(-) diff --git a/code/index/index-forward/java/nu/marginalia/index/forward/construction/ForwardIndexConverter.java b/code/index/index-forward/java/nu/marginalia/index/forward/construction/ForwardIndexConverter.java index 40edf4aa..f58ac876 100644 --- a/code/index/index-forward/java/nu/marginalia/index/forward/construction/ForwardIndexConverter.java +++ b/code/index/index-forward/java/nu/marginalia/index/forward/construction/ForwardIndexConverter.java @@ -86,7 +86,7 @@ public class ForwardIndexConverter { ByteBuffer workArea = ByteBuffer.allocate(65536); for (var instance : journal.pages()) { - try (var slopTable = new SlopTable(instance.page())) + try (var slopTable = new SlopTable(instance.baseDir(), instance.page())) { var docIdReader = instance.openCombinedId(slopTable); var metaReader = instance.openDocumentMeta(slopTable); @@ -152,7 +152,7 @@ public class ForwardIndexConverter { Roaring64Bitmap rbm = new Roaring64Bitmap(); for (var instance : journalReader.pages()) { - try (var slopTable = new SlopTable(instance.page())) { + try (var slopTable = new SlopTable(instance.baseDir(), instance.page())) { LongColumn.Reader idReader = instance.openCombinedId(slopTable); while (idReader.hasRemaining()) { diff --git a/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalPage.java b/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalPage.java index 173c6f8d..cb1bbf4d 100644 --- a/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalPage.java +++ b/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalPage.java @@ -31,39 +31,39 @@ public record IndexJournalPage(Path baseDir, int page) { } public LongColumn.Reader openCombinedId(SlopTable table) throws IOException { - return combinedId.open(table, baseDir); + return combinedId.open(table); } public LongColumn.Reader openDocumentMeta(SlopTable table) throws IOException { - return documentMeta.open(table, baseDir); + return documentMeta.open(table); } public IntColumn.Reader openFeatures(SlopTable table) throws IOException { - return features.open(table, baseDir); + return features.open(table); } public IntColumn.Reader openSize(SlopTable table) throws IOException { - return size.open(table, baseDir); + return size.open(table); } public LongArrayColumn.Reader openTermIds(SlopTable table) throws IOException { - return termIds.open(table, baseDir); + return termIds.open(table); } public ByteArrayColumn.Reader openTermMetadata(SlopTable table) throws IOException { - return termMeta.open(table, baseDir); + return termMeta.open(table); } public GammaCodedSequenceArrayColumn.Reader openTermPositions(SlopTable table) throws IOException { - return positions.open(table, baseDir); + return positions.open(table); } public GammaCodedSequenceArrayColumn.Reader openSpans(SlopTable table) throws IOException { - return spans.open(table, baseDir); + return spans.open(table); } public ByteArrayColumn.Reader openSpanCodes(SlopTable table) throws IOException { - return spanCodes.open(table, baseDir); + return spanCodes.open(table); } } diff --git a/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalSlopWriter.java b/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalSlopWriter.java index a62001e0..c04fab0d 100644 --- a/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalSlopWriter.java +++ b/code/index/index-journal/java/nu/marginalia/index/journal/IndexJournalSlopWriter.java @@ -33,24 +33,24 @@ public class IndexJournalSlopWriter extends SlopTable { public IndexJournalSlopWriter(Path dir, int page) throws IOException { - super(page); + super(dir, page); if (!Files.exists(dir)) { Files.createDirectory(dir); } - featuresWriter = IndexJournalPage.features.create(this, dir); - sizeWriter = IndexJournalPage.size.create(this, dir); + featuresWriter = IndexJournalPage.features.create(this); + sizeWriter = IndexJournalPage.size.create(this); - combinedIdWriter = IndexJournalPage.combinedId.create(this, dir); - documentMetaWriter = IndexJournalPage.documentMeta.create(this, dir); + combinedIdWriter = IndexJournalPage.combinedId.create(this); + documentMetaWriter = IndexJournalPage.documentMeta.create(this); - termIdsWriter = IndexJournalPage.termIds.create(this, dir); - termMetadataWriter = IndexJournalPage.termMeta.create(this, dir); - termPositionsWriter = IndexJournalPage.positions.create(this, dir); + termIdsWriter = IndexJournalPage.termIds.create(this); + termMetadataWriter = IndexJournalPage.termMeta.create(this); + termPositionsWriter = IndexJournalPage.positions.create(this); - spanCodesWriter = IndexJournalPage.spanCodes.create(this, dir); - spansWriter = IndexJournalPage.spans.create(this, dir); + spanCodesWriter = IndexJournalPage.spanCodes.create(this); + spansWriter = IndexJournalPage.spans.create(this); } @SneakyThrows diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexDocuments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexDocuments.java index 09ffd54a..02055c7f 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexDocuments.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexDocuments.java @@ -68,7 +68,7 @@ public class FullPreindexDocuments { private static void createUnsortedDocsFile(Path docsFile, Path workDir, - IndexJournalPage journalInstance, + IndexJournalPage instance, FullPreindexWordSegments segments, DocIdRewriter docIdRewriter) throws IOException { @@ -77,12 +77,12 @@ public class FullPreindexDocuments { final ByteBuffer tempBuffer = ByteBuffer.allocate(1024*1024*100); try (var assembly = RandomFileAssembler.create(workDir, fileSizeLongs); - var slopTable = new SlopTable(journalInstance.page())) + var slopTable = new SlopTable(instance.baseDir(), instance.page())) { - var docIds = journalInstance.openCombinedId(slopTable); - var termIds = journalInstance.openTermIds(slopTable); - var termMeta = journalInstance.openTermMetadata(slopTable); - var positions = journalInstance.openTermPositions(slopTable); + var docIds = instance.openCombinedId(slopTable); + var termIds = instance.openTermIds(slopTable); + var termMeta = instance.openTermMetadata(slopTable); + var positions = instance.openTermPositions(slopTable); var offsetMap = segments.asMap(RECORD_SIZE_LONGS); offsetMap.defaultReturnValue(0); diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexWordSegments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexWordSegments.java index bddbba7c..0a4e39a7 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexWordSegments.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/full/FullPreindexWordSegments.java @@ -52,7 +52,7 @@ public class FullPreindexWordSegments { return ret; } - public static FullPreindexWordSegments construct(IndexJournalPage journalInstance, + public static FullPreindexWordSegments construct(IndexJournalPage instance, Path wordIdsFile, Path countsFile) throws IOException @@ -60,8 +60,8 @@ public class FullPreindexWordSegments { Long2IntOpenHashMap countsMap = new Long2IntOpenHashMap(100_000, 0.75f); countsMap.defaultReturnValue(0); - try (var slopTable = new SlopTable(journalInstance.page())) { - var termIds = journalInstance.openTermIds(slopTable); + try (var slopTable = new SlopTable(instance.baseDir(), instance.page())) { + var termIds = instance.openTermIds(slopTable); while (termIds.hasRemaining()) { long[] tids = termIds.get(); for (long termId : tids) { diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexDocuments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexDocuments.java index 93134e87..d9290e14 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexDocuments.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexDocuments.java @@ -58,18 +58,18 @@ public class PrioPreindexDocuments { private static void createUnsortedDocsFile(Path docsFile, Path workDir, - IndexJournalPage journalInstance, + IndexJournalPage instance, PrioPreindexWordSegments segments, DocIdRewriter docIdRewriter) throws IOException { long fileSizeLongs = RECORD_SIZE_LONGS * segments.totalSize(); try (var assembly = RandomFileAssembler.create(workDir, fileSizeLongs); - var slopTable = new SlopTable(journalInstance.page())) + var slopTable = new SlopTable(instance.baseDir(), instance.page())) { - var docIds = journalInstance.openCombinedId(slopTable); - var termIds = journalInstance.openTermIds(slopTable); - var termMeta = journalInstance.openTermMetadata(slopTable); + var docIds = instance.openCombinedId(slopTable); + var termIds = instance.openTermIds(slopTable); + var termMeta = instance.openTermMetadata(slopTable); var offsetMap = segments.asMap(RECORD_SIZE_LONGS); offsetMap.defaultReturnValue(0); diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexWordSegments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexWordSegments.java index 8814a434..69c5ea61 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexWordSegments.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioPreindexWordSegments.java @@ -52,7 +52,7 @@ public class PrioPreindexWordSegments { return ret; } - public static PrioPreindexWordSegments construct(IndexJournalPage journalInstance, + public static PrioPreindexWordSegments construct(IndexJournalPage instance, Path wordIdsFile, Path countsFile) throws IOException @@ -60,9 +60,9 @@ public class PrioPreindexWordSegments { Long2IntOpenHashMap countsMap = new Long2IntOpenHashMap(100_000, 0.75f); countsMap.defaultReturnValue(0); - try (var slopTable = new SlopTable(journalInstance.page())) { - var termIds = journalInstance.openTermIds(slopTable); - var termMetas = journalInstance.openTermMetadata(slopTable); + try (var slopTable = new SlopTable(instance.baseDir(), instance.page())) { + var termIds = instance.openTermIds(slopTable); + var termMetas = instance.openTermMetadata(slopTable); while (termIds.hasRemaining()) { long[] data = termIds.get(); diff --git a/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDocumentRecord.java b/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDocumentRecord.java index 6e03976c..4c6b62dd 100644 --- a/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDocumentRecord.java +++ b/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDocumentRecord.java @@ -153,19 +153,19 @@ public record SlopDocumentRecord( } public KeywordsProjectionReader(Path baseDir, int page) throws IOException { - super(page); - domainsReader = domainsColumn.open(this, baseDir); - ordinalsReader = ordinalsColumn.open(this, baseDir); - htmlFeaturesReader = htmlFeaturesColumn.open(this, baseDir); - domainMetadataReader = domainMetadata.open(this, baseDir); - lengthsReader = lengthsColumn.open(this, baseDir); + super(baseDir, page); + domainsReader = domainsColumn.open(this); + ordinalsReader = ordinalsColumn.open(this); + htmlFeaturesReader = htmlFeaturesColumn.open(this); + domainMetadataReader = domainMetadata.open(this); + lengthsReader = lengthsColumn.open(this); - keywordsReader = keywordsColumn.open(this, baseDir); - termMetaReader = termMetaColumn.open(this, baseDir); - termPositionsReader = termPositionsColumn.open(this, baseDir); + keywordsReader = keywordsColumn.open(this); + termMetaReader = termMetaColumn.open(this); + termPositionsReader = termPositionsColumn.open(this); - spanCodesReader = spanCodesColumn.open(this, baseDir); - spansReader = spansColumn.open(this, baseDir); + spanCodesReader = spanCodesColumn.open(this); + spansReader = spansColumn.open(this); } public boolean hasMore() throws IOException { @@ -221,19 +221,19 @@ public record SlopDocumentRecord( } public MetadataReader(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - this.domainsReader = domainsColumn.open(this, baseDir); - this.urlsReader = urlsColumn.open(this, baseDir); - this.ordinalsReader = ordinalsColumn.open(this, baseDir); - this.titlesReader = titlesColumn.open(this, baseDir); - this.descriptionsReader = descriptionsColumn.open(this, baseDir); - this.htmlFeaturesReader = htmlFeaturesColumn.open(this, baseDir); - this.htmlStandardsReader = htmlStandardsColumn.open(this, baseDir); - this.lengthsReader = lengthsColumn.open(this, baseDir); - this.hashesReader = hashesColumn.open(this, baseDir); - this.qualitiesReader = qualitiesColumn.open(this, baseDir); - this.pubYearReader = pubYearColumn.open(this, baseDir); + this.domainsReader = domainsColumn.open(this); + this.urlsReader = urlsColumn.open(this); + this.ordinalsReader = ordinalsColumn.open(this); + this.titlesReader = titlesColumn.open(this); + this.descriptionsReader = descriptionsColumn.open(this); + this.htmlFeaturesReader = htmlFeaturesColumn.open(this); + this.htmlStandardsReader = htmlStandardsColumn.open(this); + this.lengthsReader = lengthsColumn.open(this); + this.hashesReader = hashesColumn.open(this); + this.qualitiesReader = qualitiesColumn.open(this); + this.pubYearReader = pubYearColumn.open(this); } public boolean hasMore() throws IOException { @@ -281,29 +281,29 @@ public record SlopDocumentRecord( private final GammaCodedSequenceArrayColumn.Writer spansWriter; public Writer(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - domainsWriter = domainsColumn.create(this, baseDir); - urlsWriter = urlsColumn.create(this, baseDir); - ordinalsWriter = ordinalsColumn.create(this, baseDir); - statesWriter = statesColumn.create(this, baseDir); - stateReasonsWriter = stateReasonsColumn.create(this, baseDir); - titlesWriter = titlesColumn.create(this, baseDir); - descriptionsWriter = descriptionsColumn.create(this, baseDir); - htmlFeaturesWriter = htmlFeaturesColumn.create(this, baseDir); - htmlStandardsWriter = htmlStandardsColumn.create(this, baseDir); - lengthsWriter = lengthsColumn.create(this, baseDir); - hashesWriter = hashesColumn.create(this, baseDir); - qualitiesWriter = qualitiesColumn.create(this, baseDir); - domainMetadataWriter = domainMetadata.create(this, baseDir); - pubYearWriter = pubYearColumn.create(this, baseDir); + domainsWriter = domainsColumn.create(this); + urlsWriter = urlsColumn.create(this); + ordinalsWriter = ordinalsColumn.create(this); + statesWriter = statesColumn.create(this); + stateReasonsWriter = stateReasonsColumn.create(this); + titlesWriter = titlesColumn.create(this); + descriptionsWriter = descriptionsColumn.create(this); + htmlFeaturesWriter = htmlFeaturesColumn.create(this); + htmlStandardsWriter = htmlStandardsColumn.create(this); + lengthsWriter = lengthsColumn.create(this); + hashesWriter = hashesColumn.create(this); + qualitiesWriter = qualitiesColumn.create(this); + domainMetadataWriter = domainMetadata.create(this); + pubYearWriter = pubYearColumn.create(this); - keywordsWriter = keywordsColumn.create(this, baseDir); - termMetaWriter = termMetaColumn.create(this, baseDir); - termPositionsWriter = termPositionsColumn.create(this, baseDir); + keywordsWriter = keywordsColumn.create(this); + termMetaWriter = termMetaColumn.create(this); + termPositionsWriter = termPositionsColumn.create(this); - spansCodesWriter = spanCodesColumn.create(this, baseDir); - spansWriter = spansColumn.create(this, baseDir); + spansCodesWriter = spanCodesColumn.create(this); + spansWriter = spansColumn.create(this); } public void write(SlopDocumentRecord record) throws IOException { diff --git a/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainLinkRecord.java b/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainLinkRecord.java index db318ae6..6d1bcd03 100644 --- a/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainLinkRecord.java +++ b/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainLinkRecord.java @@ -28,10 +28,10 @@ public record SlopDomainLinkRecord( } public Reader(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - sourcesReader = sourcesColumn.open(this, baseDir); - destsReader = destsColumn.open(this, baseDir); + sourcesReader = sourcesColumn.open(this); + destsReader = destsColumn.open(this); } public boolean hasMore() throws IOException { @@ -58,10 +58,10 @@ public record SlopDomainLinkRecord( private final TxtStringColumn.Writer destsWriter; public Writer(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - sourcesWriter = sourcesColumn.create(this, baseDir); - destsWriter = destsColumn.create(this, baseDir); + sourcesWriter = sourcesColumn.create(this); + destsWriter = destsColumn.create(this); } public void write(SlopDomainLinkRecord record) throws IOException { diff --git a/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainRecord.java b/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainRecord.java index f0345df1..6cb924f2 100644 --- a/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainRecord.java +++ b/code/processes/converting-process/model/java/nu/marginalia/model/processed/SlopDomainRecord.java @@ -48,9 +48,9 @@ public record SlopDomainRecord( } public DomainNameReader(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - domainsReader = domainsColumn.open(this, baseDir); + domainsReader = domainsColumn.open(this); } public boolean hasMore() throws IOException { @@ -71,10 +71,10 @@ public record SlopDomainRecord( } public DomainWithIpReader(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - domainsReader = domainsColumn.open(this, baseDir); - ipReader = ipColumn.open(this, baseDir); + domainsReader = domainsColumn.open(this); + ipReader = ipColumn.open(this); } public boolean hasMore() throws IOException { @@ -107,18 +107,18 @@ public record SlopDomainRecord( } public Reader(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - domainsReader = domainsColumn.open(this, baseDir); - statesReader = statesColumn.open(this, baseDir); - redirectReader = redirectDomainsColumn.open(this, baseDir); - ipReader = ipColumn.open(this, baseDir); + domainsReader = domainsColumn.open(this); + statesReader = statesColumn.open(this); + redirectReader = redirectDomainsColumn.open(this); + ipReader = ipColumn.open(this); - knownUrlsReader = knownUrlsColumn.open(this, baseDir); - goodUrlsReader = goodUrlsColumn.open(this, baseDir); - visitedUrlsReader = visitedUrlsColumn.open(this, baseDir); + knownUrlsReader = knownUrlsColumn.open(this); + goodUrlsReader = goodUrlsColumn.open(this); + visitedUrlsReader = visitedUrlsColumn.open(this); - rssFeedsReader = rssFeedsColumn.open(this, baseDir); + rssFeedsReader = rssFeedsColumn.open(this); } public boolean hasMore() throws IOException { @@ -158,18 +158,18 @@ public record SlopDomainRecord( private final ObjectArrayColumn.Writer rssFeedsWriter; public Writer(Path baseDir, int page) throws IOException { - super(page); + super(baseDir, page); - domainsWriter = domainsColumn.create(this, baseDir); - statesWriter = statesColumn.create(this, baseDir); - redirectWriter = redirectDomainsColumn.create(this, baseDir); - ipWriter = ipColumn.create(this, baseDir); + domainsWriter = domainsColumn.create(this); + statesWriter = statesColumn.create(this); + redirectWriter = redirectDomainsColumn.create(this); + ipWriter = ipColumn.create(this); - knownUrlsWriter = knownUrlsColumn.create(this, baseDir); - goodUrlsWriter = goodUrlsColumn.create(this, baseDir); - visitedUrlsWriter = visitedUrlsColumn.create(this, baseDir); + knownUrlsWriter = knownUrlsColumn.create(this); + goodUrlsWriter = goodUrlsColumn.create(this); + visitedUrlsWriter = visitedUrlsColumn.create(this); - rssFeedsWriter = rssFeedsColumn.create(this, baseDir); + rssFeedsWriter = rssFeedsColumn.create(this); } public void write(SlopDomainRecord record) throws IOException { diff --git a/settings.gradle b/settings.gradle index 015cd688..ccd51ccb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -226,7 +226,7 @@ dependencyResolutionManagement { library('jetty-util','org.eclipse.jetty','jetty-util').version('9.4.54.v20240208') library('jetty-servlet','org.eclipse.jetty','jetty-servlet').version('9.4.54.v20240208') - library('slop', 'nu.marginalia', 'slop').version('0.0.4-SNAPSHOT') + library('slop', 'nu.marginalia', 'slop').version('0.0.5-SNAPSHOT') bundle('jetty', ['jetty-server', 'jetty-util', 'jetty-servlet'])