diff --git a/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexEntrySource.java b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexEntrySource.java index 851bf9ab..f10ddb1c 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexEntrySource.java +++ b/code/index/index-reverse/java/nu/marginalia/index/ReverseIndexEntrySource.java @@ -1,6 +1,6 @@ package nu.marginalia.index; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.btree.BTreeReader; import nu.marginalia.index.query.EntrySource; diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/CountToOffsetTransformer.java b/code/index/index-reverse/java/nu/marginalia/index/construction/CountToOffsetTransformer.java index 29f775f7..d9aed10d 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/CountToOffsetTransformer.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/CountToOffsetTransformer.java @@ -1,6 +1,6 @@ package nu.marginalia.index.construction; -import nu.marginalia.array.functional.LongTransformer; +import nu.marginalia.array.algo.LongArrayTransformations; /** * Transforms an array of item-counts into an array of item-offsets such that the previous counts would fit into an @@ -9,7 +9,7 @@ import nu.marginalia.array.functional.LongTransformer; * [ 1, 2, 3, 5, ... ] -> [ 0, 1, 3, 6, 11, ... ] * */ -public class CountToOffsetTransformer implements LongTransformer { +public class CountToOffsetTransformer implements LongArrayTransformations.LongTransformer { long offset = 0; public final int entrySize; diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/IndexSizeEstimator.java b/code/index/index-reverse/java/nu/marginalia/index/construction/IndexSizeEstimator.java index 566b14b5..32047720 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/IndexSizeEstimator.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/IndexSizeEstimator.java @@ -1,13 +1,13 @@ package nu.marginalia.index.construction; -import nu.marginalia.array.functional.LongBinaryOperation; +import nu.marginalia.array.algo.LongArrayTransformations; import nu.marginalia.btree.model.BTreeContext; /** Calculates the necessary size of an index from an array of offsets (@see CountToOffsetTransformer)

* * Used with LongArray.fold() * */ -public class IndexSizeEstimator implements LongBinaryOperation { +public class IndexSizeEstimator implements LongArrayTransformations.LongBinaryOperation { private final BTreeContext bTreeContext; private final int entrySize; diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java index 4ace48a9..dd5499bf 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/ReverseIndexBTreeTransformer.java @@ -1,7 +1,7 @@ package nu.marginalia.index.construction; import nu.marginalia.array.LongArray; -import nu.marginalia.array.functional.LongIOTransformer; +import nu.marginalia.array.algo.LongArrayTransformations; import nu.marginalia.btree.BTreeWriter; import nu.marginalia.btree.model.BTreeContext; @@ -9,7 +9,7 @@ import java.io.IOException; import java.nio.channels.FileChannel; /** Constructs the BTrees in a reverse index */ -public class ReverseIndexBTreeTransformer implements LongIOTransformer { +public class ReverseIndexBTreeTransformer implements LongArrayTransformations.LongIOTransformer { private final BTreeWriter writer; private final FileChannel intermediateChannel; diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java index 04303210..0e6c32fb 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/ReversePreindexWordSegments.java @@ -72,7 +72,7 @@ public class ReversePreindexWordSegments { } // Sort the words file - words.quickSort(0, counts.size()); + words.sort(0, counts.size()); // Populate the counts for (i = 0; i < countsMap.size(); i++) { diff --git a/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java b/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java index c10a7845..fb261ed3 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java +++ b/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRejectFilter.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.btree.BTreeReader; import nu.marginalia.index.query.filter.QueryFilterStepIf; diff --git a/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java b/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java index bde2ccc8..aa6638a9 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java +++ b/code/index/index-reverse/java/nu/marginalia/index/query/ReverseIndexRetainFilter.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.btree.BTreeReader; import nu.marginalia.index.query.filter.QueryFilterStepIf; diff --git a/code/index/index-reverse/test/nu/marginalia/index/ReverseIndexReaderTest.java b/code/index/index-reverse/test/nu/marginalia/index/ReverseIndexReaderTest.java index ed8b4193..265864c4 100644 --- a/code/index/index-reverse/test/nu/marginalia/index/ReverseIndexReaderTest.java +++ b/code/index/index-reverse/test/nu/marginalia/index/ReverseIndexReaderTest.java @@ -1,7 +1,6 @@ package nu.marginalia.index; -import nu.marginalia.array.algo.SortingContext; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.index.construction.DocIdRewriter; import nu.marginalia.index.construction.ReversePreindex; import nu.marginalia.index.construction.TestJournalFactory; @@ -22,14 +21,12 @@ import static org.junit.jupiter.api.Assertions.*; class ReverseIndexReaderTest { TestJournalFactory journalFactory; Path tempDir; - SortingContext sortingContext; @BeforeEach public void setUp() throws IOException { journalFactory = new TestJournalFactory(); tempDir = Files.createTempDirectory("sort"); - sortingContext = new SortingContext(Path.of("invalid"), 1<<20); } @AfterEach @@ -64,7 +61,6 @@ class ReverseIndexReaderTest { var indexReader = createIndex( new EntryDataWithWordMeta(100, 101, wm(50, 51), wm(51, 52)), new EntryDataWithWordMeta(101, 101, wm(51, 53), wm(52, 54)) - ); assertEquals(1, indexReader.numDocuments(50)); diff --git a/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java index 049c9bad..3f742897 100644 --- a/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java +++ b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java @@ -2,7 +2,6 @@ package nu.marginalia.index.construction; import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.array.algo.SortingContext; import nu.marginalia.btree.BTreeReader; import nu.marginalia.btree.model.BTreeHeader; import org.junit.jupiter.api.AfterEach; @@ -24,7 +23,6 @@ class ReversePreindexFinalizeTest { Path wordsIdFile; Path docsFile; Path tempDir; - SortingContext sortingContext; @BeforeEach public void setUp() throws IOException { @@ -34,7 +32,6 @@ class ReversePreindexFinalizeTest { wordsIdFile = Files.createTempFile("words", ".dat"); docsFile = Files.createTempFile("docs", ".dat"); tempDir = Files.createTempDirectory("sort"); - sortingContext = new SortingContext(Path.of("invalid"), 1<<20); } @AfterEach diff --git a/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexMergeTest.java b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexMergeTest.java index 5d9d42f2..1a173d9a 100644 --- a/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexMergeTest.java +++ b/code/index/index-reverse/test/nu/marginalia/index/construction/ReversePreindexMergeTest.java @@ -1,7 +1,6 @@ package nu.marginalia.index.construction; -import nu.marginalia.array.algo.SortingContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -20,7 +19,6 @@ class ReversePreindexMergeTest { Path wordsIdFile; Path docsFile; Path tempDir; - SortingContext sortingContext; @BeforeEach public void setUp() throws IOException { @@ -30,7 +28,6 @@ class ReversePreindexMergeTest { wordsIdFile = Files.createTempFile("words", ".dat"); docsFile = Files.createTempFile("docs", ".dat"); tempDir = Files.createTempDirectory("sort"); - sortingContext = new SortingContext(Path.of("invalid"), 1<<20); } @AfterEach diff --git a/code/index/java/nu/marginalia/index/IndexGrpcService.java b/code/index/java/nu/marginalia/index/IndexGrpcService.java index 806b79d4..07199f9e 100644 --- a/code/index/java/nu/marginalia/index/IndexGrpcService.java +++ b/code/index/java/nu/marginalia/index/IndexGrpcService.java @@ -14,10 +14,7 @@ import nu.marginalia.api.searchquery.model.compiled.CompiledQueryLong; import nu.marginalia.api.searchquery.model.compiled.CqDataInt; import nu.marginalia.api.searchquery.model.query.SearchSpecification; import nu.marginalia.api.searchquery.model.results.*; -import nu.marginalia.api.searchquery.model.results.debug.ResultRankingDetails; -import nu.marginalia.api.searchquery.model.results.debug.ResultRankingInputs; -import nu.marginalia.api.searchquery.model.results.debug.ResultRankingOutputs; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.index.index.StatefulIndex; import nu.marginalia.index.model.SearchParameters; import nu.marginalia.index.model.SearchTerms; diff --git a/code/index/query/java/nu/marginalia/index/query/EmptyEntrySource.java b/code/index/query/java/nu/marginalia/index/query/EmptyEntrySource.java index 62e06f5f..d8e4100e 100644 --- a/code/index/query/java/nu/marginalia/index/query/EmptyEntrySource.java +++ b/code/index/query/java/nu/marginalia/index/query/EmptyEntrySource.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; /** Dummy EntrySource that returns no entries. */ public class EmptyEntrySource implements EntrySource { diff --git a/code/index/query/java/nu/marginalia/index/query/EntrySource.java b/code/index/query/java/nu/marginalia/index/query/EntrySource.java index 9c643e2b..4b0f6405 100644 --- a/code/index/query/java/nu/marginalia/index/query/EntrySource.java +++ b/code/index/query/java/nu/marginalia/index/query/EntrySource.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; /** An EntrySource is a source of entries for a query. */ diff --git a/code/index/query/java/nu/marginalia/index/query/IndexQuery.java b/code/index/query/java/nu/marginalia/index/query/IndexQuery.java index 651a3f24..81136e91 100644 --- a/code/index/query/java/nu/marginalia/index/query/IndexQuery.java +++ b/code/index/query/java/nu/marginalia/index/query/IndexQuery.java @@ -1,7 +1,7 @@ package nu.marginalia.index.query; import nu.marginalia.index.query.filter.QueryFilterStepIf; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import java.util.ArrayList; import java.util.List; diff --git a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAllOf.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAllOf.java index e9725179..8e99ae66 100644 --- a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAllOf.java +++ b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAllOf.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query.filter; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import java.util.ArrayList; import java.util.List; diff --git a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java index bea62194..7d16414a 100644 --- a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java +++ b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query.filter; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import java.util.ArrayList; import java.util.List; @@ -60,7 +60,7 @@ public class QueryFilterAnyOf implements QueryFilterStepIf { for (var step : steps) { var slice = buffer.slice(start, endOfValidData); - slice.data.quickSort(0, slice.size()); + slice.data.sort(0, slice.size()); step.apply(slice); start += slice.end; @@ -73,7 +73,7 @@ public class QueryFilterAnyOf implements QueryFilterStepIf { // After all filters have been applied, we must re-sort all the retained data // to uphold the sortedness contract - buffer.data.quickSort(0, buffer.end); + buffer.data.sort(0, buffer.end); } public String describe() { diff --git a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java index 77f503cf..e3b24128 100644 --- a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java +++ b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query.filter; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; public class QueryFilterLetThrough implements QueryFilterStepIf { diff --git a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java index 502e7c4c..4dfdb280 100644 --- a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java +++ b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query.filter; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; public class QueryFilterNoPass implements QueryFilterStepIf { static final QueryFilterStepIf instance = new QueryFilterNoPass(); diff --git a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java index 748fdde6..4f5f3a4d 100644 --- a/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java +++ b/code/index/query/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query.filter; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; public interface QueryFilterStepIf extends Comparable { boolean test(long value); diff --git a/code/index/query/test/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java b/code/index/query/test/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java index b2ef1bdb..fbe8f135 100644 --- a/code/index/query/test/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java +++ b/code/index/query/test/nu/marginalia/index/query/filter/QueryFilterStepIfTest.java @@ -1,6 +1,6 @@ package nu.marginalia.index.query.filter; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/code/libraries/array/java/nu/marginalia/array/ArrayRangeReference.java b/code/libraries/array/java/nu/marginalia/array/ArrayRangeReference.java deleted file mode 100644 index d18fe3f0..00000000 --- a/code/libraries/array/java/nu/marginalia/array/ArrayRangeReference.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.array; - -/** A reference to a range of an array. Use this class judiciously to avoid - * gc churn. - */ -public record ArrayRangeReference(T array, long start, long end) { -} diff --git a/code/libraries/array/java/nu/marginalia/array/IntArray.java b/code/libraries/array/java/nu/marginalia/array/IntArray.java deleted file mode 100644 index 355dc2dc..00000000 --- a/code/libraries/array/java/nu/marginalia/array/IntArray.java +++ /dev/null @@ -1,32 +0,0 @@ -package nu.marginalia.array; - -import nu.marginalia.array.algo.IntArrayBase; -import nu.marginalia.array.algo.IntArraySearch; -import nu.marginalia.array.algo.IntArraySort; -import nu.marginalia.array.algo.IntArrayTransformations; -import nu.marginalia.array.delegate.ShiftedIntArray; -import nu.marginalia.array.page.SegmentIntArray; - -import java.lang.foreign.Arena; - -public interface IntArray extends IntArrayBase, IntArrayTransformations, IntArraySearch, IntArraySort { - int WORD_SIZE = 4; - - static IntArray allocate(long size) { - return SegmentIntArray.onHeap(Arena.ofShared(), size); - } - - default IntArray shifted(long offset) { - return new ShiftedIntArray(offset, this); - } - default IntArray range(long start, long end) { - return new ShiftedIntArray(start, end, this); - } - - /** Translate the range into the equivalent range in the underlying array if they are in the same page */ - ArrayRangeReference directRangeIfPossible(long start, long end); - - void force(); - - default void close() { } -} diff --git a/code/libraries/array/java/nu/marginalia/array/LongArray.java b/code/libraries/array/java/nu/marginalia/array/LongArray.java index b597900c..a648dad8 100644 --- a/code/libraries/array/java/nu/marginalia/array/LongArray.java +++ b/code/libraries/array/java/nu/marginalia/array/LongArray.java @@ -4,7 +4,6 @@ import nu.marginalia.array.algo.LongArrayBase; import nu.marginalia.array.algo.LongArraySearch; import nu.marginalia.array.algo.LongArraySort; import nu.marginalia.array.algo.LongArrayTransformations; -import nu.marginalia.array.delegate.ShiftedLongArray; import nu.marginalia.array.page.UnsafeLongArray; import java.lang.foreign.Arena; @@ -19,16 +18,12 @@ public interface LongArray extends LongArrayBase, LongArrayTransformations, Long return UnsafeLongArray.onHeap(Arena.ofShared(), size); } - default LongArray shifted(long offset) { - return new ShiftedLongArray(offset, this); - } - default LongArray range(long start, long end) { - return new ShiftedLongArray(start, end, this); - } - - /** Translate the range into the equivalent range in the underlying array if they are in the same page */ - ArrayRangeReference directRangeIfPossible(long start, long end); + LongArray shifted(long offset); + LongArray range(long start, long end); + /** Force any changes to be written to the backing store */ void force(); + + /** Close the array and release any resources */ void close(); } diff --git a/code/libraries/array/java/nu/marginalia/array/algo/IntArrayBase.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArrayBase.java deleted file mode 100644 index 56826cc7..00000000 --- a/code/libraries/array/java/nu/marginalia/array/algo/IntArrayBase.java +++ /dev/null @@ -1,67 +0,0 @@ -package nu.marginalia.array.algo; - -import java.io.IOException; -import java.nio.IntBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Path; - -public interface IntArrayBase extends BulkTransferArray { - int get(long pos); - - void set(long pos, int value); - default void set(long pos, int... value) { - for (int i = 0; i < value.length; i++) { - set(pos+i, value[i]); - } - } - - long size(); - - default void fill(long start, long end, int val) { - for (long v = start; v < end; v++) { - set(v, val); - } - } - - default void swap(long pos1, long pos2) { - int tmp = get(pos1); - set(pos1, get(pos2)); - set(pos2, tmp); - } - - default void increment(long pos) { - set(pos, get(pos) + 1); - } - - default int getAndIncrement(long pos) { - int val = get(pos); - set(pos, val + 1); - return val; - } - - default void set(long start, long end, IntBuffer buffer, int bufferStart) { - for (int i = 0; i < (end-start); i++) { - set(start+i, buffer.get(i + bufferStart)); - } - } - - default void get(long start, long end, IntBuffer buffer, int bufferStart) { - for (int i = 0; i < (end-start); i++) { - buffer.put(i + bufferStart, get(start + i)); - } - } - - default void get(long start, IntBuffer buffer) { - get(start, start + buffer.remaining(), buffer, buffer.position()); - } - - default void get(long start, long end, int[] buffer) { - for (int i = 0; i < (end-start); i++) { - buffer[i] = get(start + i); - } - } - - void write(Path file) throws IOException; - - void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException; -} diff --git a/code/libraries/array/java/nu/marginalia/array/algo/IntArraySearch.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArraySearch.java deleted file mode 100644 index 6a148846..00000000 --- a/code/libraries/array/java/nu/marginalia/array/algo/IntArraySearch.java +++ /dev/null @@ -1,127 +0,0 @@ -package nu.marginalia.array.algo; - -import nu.marginalia.array.buffer.IntQueryBuffer; - -public interface IntArraySearch extends IntArrayBase { - - int LINEAR_SEARCH_CUTOFF = 64; - - default long linearSearch(int key, long fromIndex, long toIndex) { - long pos; - - for (pos = fromIndex; pos < toIndex; pos++) { - int val = get(pos); - - if (val == key) return pos; - if (val > key) break; - } - - return encodeSearchMiss(1, pos - 1); - } - - default long binarySearch(int key, long fromIndex, long toIndex) { - long low = 0; - long high = (toIndex - fromIndex) - 1; - - while (high - low >= LINEAR_SEARCH_CUTOFF) { - long mid = (low + high) >>> 1; - long midVal = get(fromIndex + mid); - - if (midVal < key) - low = mid + 1; - else if (midVal > key) - high = mid - 1; - else - return fromIndex + mid; - } - return linearSearch(key, fromIndex + low, fromIndex + high + 1); - } - - default long binarySearchUpperBound(int key, long fromIndex, long toIndex) { - long low = 0; - long high = (toIndex - fromIndex) - 1; - - while (high - low >= LINEAR_SEARCH_CUTOFF) { - long mid = (low + high) >>> 1; - long midVal = get(fromIndex + mid); - - if (midVal < key) - low = mid + 1; - else if (midVal > key) - high = mid - 1; - else - return fromIndex + mid; - } - - for (fromIndex += low; fromIndex < toIndex; fromIndex++) { - if (get(fromIndex) >= key) return fromIndex; - } - - return toIndex; - } - - - default void retain(IntQueryBuffer buffer, long boundary, long searchStart, long searchEnd) { - - if (searchStart >= searchEnd) return; - - int bv = buffer.currentValue(); - int av = get(searchStart); - long pos = searchStart; - - while (bv <= boundary && buffer.hasMore()) { - if (bv < av) { - if (!buffer.rejectAndAdvance()) break; - bv = buffer.currentValue(); - continue; - } - else if (bv == av) { - if (!buffer.retainAndAdvance()) break; - bv = buffer.currentValue(); - continue; - } - - if (++pos < searchEnd) { - av = get(pos); - } - else { - break; - } - } - } - - default void reject(IntQueryBuffer buffer, long boundary, long searchStart, long searchEnd) { - - if (searchStart >= searchEnd) return; - - int bv = buffer.currentValue(); - int av = get(searchStart); - long pos = searchStart; - - while (bv <= boundary && buffer.hasMore()) { - if (bv < av) { - if (!buffer.retainAndAdvance()) break; - bv = buffer.currentValue(); - continue; - } - else if (bv == av) { - if (!buffer.rejectAndAdvance()) break; - bv = buffer.currentValue(); - continue; - } - - if (++pos < searchEnd) { - av = get(pos); - } - else { - break; - } - } - - } - - static long encodeSearchMiss(int entrySize, long value) { - return -entrySize - Math.max(0, value); - } - -} diff --git a/code/libraries/array/java/nu/marginalia/array/algo/IntArraySort.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArraySort.java deleted file mode 100644 index 5eec50ef..00000000 --- a/code/libraries/array/java/nu/marginalia/array/algo/IntArraySort.java +++ /dev/null @@ -1,79 +0,0 @@ -package nu.marginalia.array.algo; - -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -public interface IntArraySort extends IntArrayBase { - - default boolean isSorted(long start, long end) { - if (start == end) return true; - - int val = get(start); - for (long i = start + 1; i < end; i++) { - int next = get(i); - if (next < val) - return false; - val = next; - } - - return true; - } - - default void sortLargeSpan(SortingContext ctx, long start, long end) throws IOException { - long size = end - start; - - if (size < ctx.memorySortLimit()) { - quickSort(start, end); - } - else { - mergeSort(start, end, ctx.tempDir()); - } - } - - default boolean isSortedN(int wordSize, long start, long end) { - if (start == end) return true; - - int val = get(start); - for (long i = start + wordSize; i < end; i+=wordSize) { - int next = get(i); - if (next < val) - return false; - val = next; - } - - return true; - } - - default void insertionSort(long start, long end) { - assert end - start < Integer.MAX_VALUE; - - SortAlgoInsertionSort._insertionSort(this, start, end); - } - - default void quickSort(long start, long end) { - if (end - start < 64) { - insertionSort(start, end); - } - else { - SortAlgoQuickSort._quickSortLH(this, start, end - 1); - } - } - - default void mergeSort(long start, long end, Path tmpDir) throws IOException { - int length = (int) (end - start); - - Path tmpFile = Files.createTempFile(tmpDir,"sort-"+start+"-"+(start+length), ".dat"); - try (var channel = (FileChannel) Files.newByteChannel(tmpFile, StandardOpenOption.WRITE, StandardOpenOption.READ)) { - var workBuffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, 4L * length).asIntBuffer(); - - SortAlgoMergeSort._mergeSort(this, start, length, workBuffer); - } - finally { - Files.delete(tmpFile); - } - } - -} diff --git a/code/libraries/array/java/nu/marginalia/array/algo/IntArrayTransformations.java b/code/libraries/array/java/nu/marginalia/array/algo/IntArrayTransformations.java deleted file mode 100644 index cbeb7308..00000000 --- a/code/libraries/array/java/nu/marginalia/array/algo/IntArrayTransformations.java +++ /dev/null @@ -1,47 +0,0 @@ -package nu.marginalia.array.algo; - -import nu.marginalia.array.functional.*; - -import java.io.IOException; - -public interface IntArrayTransformations extends IntArrayBase { - - default void forEach(long start, long end, LongIntConsumer consumer) { - for (long i = start; i < end; i++) { - consumer.accept(i, get(i)); - } - } - - default void transformEach(long start, long end, IntTransformer transformer) { - for (long i = start; i < end; i++) { - set(i, transformer.transform(i, get(i))); - } - } - - default void transformEachIO(long start, long end, IntIOTransformer transformer) throws IOException { - for (long i = start; i < end; i++) { - set(i, transformer.transform(i, get(i))); - } - } - - default int foldIO(int zero, long start, long end, IntBinaryIOOperation operator) throws IOException { - int accumulator = zero; - - for (long i = start; i < end; i++) { - accumulator = operator.apply(accumulator, get(i)); - } - - return accumulator; - } - - default int fold(int zero, long start, long end, IntBinaryOperation operator) { - int accumulator = zero; - - for (long i = start; i < end; i++) { - accumulator = operator.apply(accumulator, get(i)); - } - - return accumulator; - } - -} diff --git a/code/libraries/array/java/nu/marginalia/array/algo/LongArrayBase.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArrayBase.java index 728f4f41..b5ef03da 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/LongArrayBase.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/LongArrayBase.java @@ -3,32 +3,42 @@ package nu.marginalia.array.algo; import nu.marginalia.array.LongArray; import java.io.IOException; +import java.lang.foreign.MemorySegment; import java.nio.LongBuffer; import java.nio.channels.FileChannel; import java.nio.file.Path; +@SuppressWarnings("preview") public interface LongArrayBase extends BulkTransferArray { + /** Get a value from the array at the specified position */ long get(long pos); + /** Set a value in the array at the specified position */ void set(long pos, long value); + + /** Return the memory segment backing the array */ + MemorySegment getMemorySegment(); + + /** Set a sequence of value in the array starting at the specified position */ default void set(long pos, long... value) { for (int i = 0; i < value.length; i++) { set(pos+i, value[i]); } } + /** Return the size of the array */ long size(); + + /** Fill the array with the specified value at the provided range */ default void fill(long start, long end, long val) { for (long v = start; v < end; v++) { set(v, val); } } - void quickSortNative(long start, long end); - void quickSortNative128(long start, long end); default void increment(long pos) { set(pos, get(pos) + 1); } @@ -48,6 +58,16 @@ public interface LongArrayBase extends BulkTransferArray { } } + default void swap2(long pos1, long pos2) { + long tmp = get(pos1); + set(pos1, get(pos2)); + set(pos2, tmp); + + tmp = get(pos1 + 1); + set(pos1 + 1, get(pos2 + 1)); + set(pos2 + 1, tmp); + } + default long getAndIncrement(long pos) { long val = get(pos); set(pos, val + 1); diff --git a/code/libraries/array/java/nu/marginalia/array/algo/LongArraySearch.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArraySearch.java index a4ac54e5..8f7b8d68 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/LongArraySearch.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/LongArraySearch.java @@ -1,6 +1,6 @@ package nu.marginalia.array.algo; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; public interface LongArraySearch extends LongArrayBase { diff --git a/code/libraries/array/java/nu/marginalia/array/algo/LongArraySort.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArraySort.java index 55108c46..90ae1f44 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/LongArraySort.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/LongArraySort.java @@ -1,6 +1,7 @@ package nu.marginalia.array.algo; import nu.marginalia.NativeAlgos; +import nu.marginalia.array.LongArray; import java.io.IOException; import java.nio.channels.FileChannel; @@ -84,33 +85,29 @@ public interface LongArraySort extends LongArrayBase { return true; } - default void insertionSort(long start, long end) { - SortAlgoInsertionSort._insertionSort(this, start, end); + static void insertionSort(LongArraySort array, long start, long end) { + SortAlgoInsertionSort._insertionSort(array, start, end); } - default void insertionSortN(int sz, long start, long end) { - SortAlgoInsertionSort._insertionSortN(this, sz, start, end); - } - - default void quickSort(long start, long end) { - if (end - start < 64) { - insertionSort(start, end); - } - else if (NativeAlgos.isAvailable) { - quickSortNative(start, end); + static void insertionSortN(LongArraySort array, int sz, long start, long end) { + if (sz == 2) { + SortAlgoInsertionSort._insertionSort2(array, start, end); } else { + SortAlgoInsertionSort._insertionSortN(array, sz, start, end); + } + } + + default void sort(long start, long end) { + if (NativeAlgos.isAvailable) { + NativeAlgos.sort(getMemorySegment(), start, end); + } else { SortAlgoQuickSort._quickSortLH(this, start, end - 1); } } - default void quickSortJava(long start, long end) { - if (end - start < 64) { - insertionSort(start, end); - } - else { - SortAlgoQuickSort._quickSortLH(this, start, end - 1); - } + static void quickSortJava(LongArray array, long start, long end) { + SortAlgoQuickSort._quickSortLH(array, start, end - 1); } default void quickSortN(int wordSize, long start, long end) { @@ -119,23 +116,39 @@ public interface LongArraySort extends LongArrayBase { if (end == start) return; - if (NativeAlgos.isAvailable && wordSize == 2) { - quickSortNative128(start, end); + if (wordSize == 2) { + if (NativeAlgos.isAvailable) { + NativeAlgos.sort128(getMemorySegment(), start, end); + } + else { + SortAlgoQuickSort._quickSortLH2(this, start, end - 2); + } } else { SortAlgoQuickSort._quickSortLHN(this, wordSize, start, end - wordSize); } } - default void quickSortJavaN(int wordSize, long start, long end) { + static void quickSortJavaN(LongArray array, int wordSize, long start, long end) { assert ((end - start) % wordSize) == 0; if (end == start) return; - SortAlgoQuickSort._quickSortLHN(this, wordSize, start, end - wordSize); + SortAlgoQuickSort._quickSortLHN(array, wordSize, start, end - wordSize); } + static void quickSortJava2(LongArray array, long start, long end) { + assert ((end - start) % 2) == 0; + + if (end == start) + return; + + SortAlgoQuickSort._quickSortLH2(array, start, end - 2); + } + + /** Don't use this method, it's slow. */ + @Deprecated default void mergeSortN(int wordSize, long start, long end, Path tmpDir) throws IOException { int length = (int) (end - start); assert (length % wordSize) == 0; @@ -152,6 +165,8 @@ public interface LongArraySort extends LongArrayBase { } + /** Don't use this method, it's slow. */ + @Deprecated default void mergeSort(long start, long end, Path tmpDir) throws IOException { int length = (int) (end - start); diff --git a/code/libraries/array/java/nu/marginalia/array/algo/LongArrayTransformations.java b/code/libraries/array/java/nu/marginalia/array/algo/LongArrayTransformations.java index 278e5cb6..e3c23b2d 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/LongArrayTransformations.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/LongArrayTransformations.java @@ -1,29 +1,34 @@ package nu.marginalia.array.algo; -import nu.marginalia.array.functional.*; - import java.io.IOException; public interface LongArrayTransformations extends LongArrayBase { + /** Applies the provided consumer to each element in the array range */ default void forEach(long start, long end, LongLongConsumer consumer) { for (long i = start; i < end; i++) { consumer.accept(i, get(i)); } } + /** Transforms each element in the array range using the provided transformer, + * so that array[i] = transformer.apply(i, array[i]) */ default void transformEach(long start, long end, LongTransformer transformer) { for (long i = start; i < end; i++) { set(i, transformer.transform(i, get(i))); } } + /** Transforms each element in the array range using the provided transformer, + * so that array[i] = transformer.apply(i, array[i]) */ default void transformEachIO(long start, long end, LongIOTransformer transformer) throws IOException { for (long i = start; i < end; i++) { set(i, transformer.transform(i, get(i))); } } + /** Transforms each element in the array range using the provided transformer, + * so that array[i] = transformer.apply(i, operator.apply(i-1, ...)) */ default long foldIO(long zero, long start, long end, LongBinaryIOOperation operator) throws IOException { long accumulator = zero; @@ -34,6 +39,8 @@ public interface LongArrayTransformations extends LongArrayBase { return accumulator; } + /** Transforms each element in the array range using the provided transformer, + * so that array[i] = transformer.apply(i, operator.apply(i-1, ...)) */ default long fold(long zero, long start, long end, LongBinaryOperation operator) { long accumulator = zero; @@ -44,4 +51,23 @@ public interface LongArrayTransformations extends LongArrayBase { return accumulator; } + interface LongBinaryIOOperation { + long apply(long left, long right) throws IOException; + } + + interface LongBinaryOperation { + long apply(long left, long right); + } + + interface LongIOTransformer { + long transform(long pos, long old) throws IOException; + } + + interface LongLongConsumer { + void accept(long pos, long val); + } + + interface LongTransformer { + long transform(long pos, long old); + } } diff --git a/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java index 5980b5ac..e1514ba3 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoInsertionSort.java @@ -2,6 +2,35 @@ package nu.marginalia.array.algo; class SortAlgoInsertionSort { + static void _insertionSort2(LongArraySort array, long start, long end) { + + assert end - start < Integer.MAX_VALUE; + + int span = (int) (end - start); + + assert (span % 2) == 0; + + if (span <= 2) { + return; + } + + long k; + long v; + + for (long i = 1; i < span / 2; i++) { + k = array.get(start + i * 2); + v = array.get(start + i * 2 + 1); + + long j; + for (j = i - 1; j >= 0 && array.get(start + j * 2) > k; j--) { + shift(array, start + j * 2, start + (j + 1) * 2, 2); + } + + array.set(start + (j + 1) * 2, k); + array.set(start + (j + 1) * 2 + 1, v); + } + } + static void _insertionSortN(LongArraySort array, int sz, long start, long end) { assert end - start < Integer.MAX_VALUE; @@ -22,7 +51,7 @@ class SortAlgoInsertionSort { long j; for (j = i - 1; j >= 0 && array.get(start + j * sz) > key; j--) { - shiftN(array, sz, start + j * sz, start + (j + 1) * sz, sz); + shift(array, start + j * sz, start + (j + 1) * sz, sz); } array.set(start + (j + 1) * sz, buf); @@ -49,44 +78,11 @@ class SortAlgoInsertionSort { array.set(start + j + 1, key); } } - static void _insertionSort(IntArraySort array, long start, long end) { - assert end - start < Integer.MAX_VALUE; - int n = (int) (end - start); - - if (n <= 1) { - return; - } - - for (int i = 1; i < n; i++) { - int key = array.get(start + i); - - int j; - for (j = i - 1; j >= 0 && array.get(start + j) > key; j--) { - shift(array, start + j, start + j + 1, 1); - } - - array.set(start + j + 1, key); - } - } - - private static void shiftN(LongArraySort array, int sz, long start, long end, long shift) { - for (long i = start; i < end; i+=sz) { - for (int j = 0; j < sz; j++) { - array.set(i + j + shift, array.get(i + j)); - } - } - } private static void shift(LongArraySort array, long start, long end, long shift) { for (long i = start; i < end; i++) { array.set(i + shift, array.get(i)); } } - private static void shift(IntArraySort array, long start, long end, long shift) { - for (long i = start; i < end; i++) { - array.set(i + shift, array.get(i)); - } - } - } diff --git a/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoMergeSort.java b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoMergeSort.java index f9f51898..846f4e65 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoMergeSort.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoMergeSort.java @@ -1,38 +1,17 @@ package nu.marginalia.array.algo; import java.io.IOException; -import java.nio.IntBuffer; import java.nio.LongBuffer; +@Deprecated class SortAlgoMergeSort { - static void _mergeSort(IntArraySort array, long start, int length, IntBuffer workBuffer) { - int width = Math.min(Integer.highestOneBit(length), 1 << 16); - - // Do in-memory sorting up until internalSortLimit first - for (int i = 0; i < length; i += width) { - array.quickSort(start + i, start + i + Math.min(width, length-i)); - } - - // Then finish with merge sort - for (width = 1; width < length; width*=2) { - - for (int i = 0; i < length; i += 2*width) { - _merge(array, start, i, Math.min(i+width, length), Math.min(i+2*width, length), workBuffer); - } - - workBuffer.clear(); - array.set(start, start + length, workBuffer, 0); - } - - } - static void _mergeSort(LongArraySort array, long start, int length, LongBuffer workBuffer) { int width = Math.min(Integer.highestOneBit(length), 1 << 16); // Do in-memory sorting up until internalSortLimit first for (int i = 0; i < length; i += width) { - array.quickSort(start + i, start + i + Math.min(width, length-i)); + array.sort(start + i, start + i + Math.min(width, length-i)); } // Then finish with merge sort @@ -110,19 +89,4 @@ class SortAlgoMergeSort { } } - static void _merge(IntArraySort array, long offset, int left, int right, int end, IntBuffer workBuffer) { - long idxL = left; - long idxR = right; - - for (int putPos = left; putPos < end; putPos++) { - if (idxL < right && (idxR >= end || array.get(offset+idxL) < array.get(offset+idxR))) { - workBuffer.put(putPos, array.get(offset+idxL)); - idxL++; - } - else { - workBuffer.put(putPos, array.get(offset+idxR)); - idxR++; - } - } - } } diff --git a/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoQuickSort.java b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoQuickSort.java index 1e70b02a..86ea7972 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoQuickSort.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/SortAlgoQuickSort.java @@ -3,53 +3,27 @@ package nu.marginalia.array.algo; class SortAlgoQuickSort { - static void _quickSortLH(IntArraySort array, long low, long highInclusive) { - + static void _quickSortLH2(LongArraySort array, long low, long highInclusive) { if (low < 0 || highInclusive < 0 || low >= highInclusive) return; - if (highInclusive - low < 32) { - array.insertionSort(low, highInclusive + 1); + if (highInclusive - low < 8) { + SortAlgoInsertionSort._insertionSort2(array, low, highInclusive + 2); return; } - long p = _quickSortPartition(array, low, highInclusive); + long p = _quickSortPartition2(array, low, highInclusive); - _quickSortLH(array, low, p); - _quickSortLH(array, p + 1, highInclusive); + _quickSortLH2(array, low, p); + _quickSortLH2(array, p + 2, highInclusive); } - - static long _quickSortPartition(IntArraySort array, long low, long high) { - - long pivotPoint = ((low + high) / (2L)); - int pivot = array.get(pivotPoint); - - long i = low - 1; - long j = high + 1; - - for (;;) { - do { - i+=1; - } while (array.get(i) < pivot); - - do { - j-=1; - } - while (array.get(j) > pivot); - - if (i >= j) return j; - else array.swap(i, j); - } - } - - static void _quickSortLHN(LongArraySort array, int wordSize, long low, long highInclusive) { if (low < 0 || highInclusive < 0 || low >= highInclusive) return; - if (highInclusive - low < 32L*wordSize) { - array.insertionSortN(wordSize, low, highInclusive + wordSize); + if (highInclusive - low < 8) { + SortAlgoInsertionSort._insertionSortN(array, wordSize, low, highInclusive + wordSize); return; } @@ -65,8 +39,8 @@ class SortAlgoQuickSort { if (low < 0 || highInclusive < 0 || low >= highInclusive) return; - if (highInclusive - low < 32) { - array.insertionSort(low, highInclusive + 1); + if (highInclusive - low < 8) { + SortAlgoInsertionSort._insertionSort(array, low, highInclusive + 1); return; } @@ -126,4 +100,30 @@ class SortAlgoQuickSort { } } + static long _quickSortPartition2(LongArraySort array, long low, long high) { + + long delta = (high - low) / (2L); + long pivotPoint = low + (delta / 2) * 2; + + long pivot = array.get(pivotPoint); + + long i = low - 2; + long j = high + 2; + + for (;;) { + do { + i+=2; + } + while (array.get(i) < pivot); + + do { + j-=2; + } + while (array.get(j) > pivot); + + if (i >= j) return j; + else array.swap2(i, j); + } + } + } diff --git a/code/libraries/array/java/nu/marginalia/array/algo/SortingContext.java b/code/libraries/array/java/nu/marginalia/array/algo/SortingContext.java deleted file mode 100644 index 84bb2007..00000000 --- a/code/libraries/array/java/nu/marginalia/array/algo/SortingContext.java +++ /dev/null @@ -1,14 +0,0 @@ -package nu.marginalia.array.algo; - -import java.nio.file.Path; - -/** - * - * @param tempDir Directory where MergeSort will allocate temporary buffers - * @param memorySortLimit Breaking point where MergeSort will be preferred over QuickSort. This is specified in - * number of items. So for e.g. long array n=2, 16 bytes x this value is the memory usage - */ -public record SortingContext( - Path tempDir, - int memorySortLimit) { -} diff --git a/code/libraries/array/java/nu/marginalia/array/algo/TwoArrayOperations.java b/code/libraries/array/java/nu/marginalia/array/algo/TwoArrayOperations.java index c4af9090..2ca5dd5d 100644 --- a/code/libraries/array/java/nu/marginalia/array/algo/TwoArrayOperations.java +++ b/code/libraries/array/java/nu/marginalia/array/algo/TwoArrayOperations.java @@ -158,23 +158,12 @@ public class TwoArrayOperations { * Count the number of distinct elements in two sorted arrays. */ public static long countDistinctElements(LongArray a, LongArray b, long aStart, long aEnd, long bStart, long bEnd) { - var directRangeA = a.directRangeIfPossible(aStart, aEnd); - var directRangeB = b.directRangeIfPossible(bStart, bEnd); - // Ensure that the arrays are sorted if (TwoArrayOperations.class.desiredAssertionStatus()) { assert (a.isSorted(aStart, aEnd)); assert (b.isSorted(bStart, bEnd)); } - a = directRangeA.array(); - aStart = directRangeA.start(); - aEnd = directRangeA.end(); - - b = directRangeB.array(); - bStart = directRangeB.start(); - bEnd = directRangeB.end(); - return countDistinctElementsDirect(a, b, aStart, aEnd, bStart, bEnd); } @@ -188,16 +177,13 @@ public class TwoArrayOperations { assert (b.isSortedN(stepSize, bStart, bEnd)); } - var directRangeA = a.directRangeIfPossible(aStart, aEnd); - var directRangeB = b.directRangeIfPossible(bStart, bEnd); - return countDistinctElementsDirectN(stepSize, - directRangeA.array(), - directRangeB.array(), - directRangeA.start(), - directRangeA.end(), - directRangeB.start(), - directRangeB.end()); + a, + b, + aStart, + aEnd, + bStart, + bEnd); } private static long countDistinctElementsDirect(LongArray a, LongArray b, long aStart, long aEnd, long bStart, long bEnd) { diff --git a/code/libraries/array/java/nu/marginalia/array/buffer/IntQueryBuffer.java b/code/libraries/array/java/nu/marginalia/array/buffer/IntQueryBuffer.java deleted file mode 100644 index bd88ccc0..00000000 --- a/code/libraries/array/java/nu/marginalia/array/buffer/IntQueryBuffer.java +++ /dev/null @@ -1,112 +0,0 @@ -package nu.marginalia.array.buffer; - -import java.util.Arrays; - -public class IntQueryBuffer { - public final int[] data; - public int end; - - private int read = 0; - private int write = 0; - - public IntQueryBuffer(int size) { - this.data = new int[size]; - this.end = size; - } - - public IntQueryBuffer(int [] data, int size) { - this.data = data; - this.end = size; - } - - public int[] copyData() { - return Arrays.copyOf(data, end); - } - - public boolean isEmpty() { - return end == 0; - } - - public int size() { - return end; - } - - public int currentValue() { - return data[read]; - } - - public boolean rejectAndAdvance() { - return ++read < end; - } - - public boolean retainAndAdvance() { - if (read != write) { - int tmp = data[write]; - data[write] = data[read]; - data[read] = tmp; - } - - write++; - - return ++read < end; - } - - public boolean hasMore() { - return read < end; - } - - public void finalizeFiltering() { - end = write; - read = 0; - write = 0; - } - - public void startFilterForRange(int pos, int end) { - read = write = pos; - this.end = end; - } - - public void reset() { - end = data.length; - read = 0; - write = 0; - } - - public void zero() { - end = 0; - read = 0; - write = 0; - Arrays.fill(data, 0); - } - - public void uniq() { - if (end <= 1) return; - - int prev = currentValue(); - retainAndAdvance(); - - while (hasMore()) { - - int val = currentValue(); - - if (prev == val) { - rejectAndAdvance(); - } else { - retainAndAdvance(); - prev = val; - } - - } - - finalizeFiltering(); - } - - public String toString() { - return getClass().getSimpleName() + "[" + - "read = " + read + - ",write = " + write + - ",end = " + end + - ",data = [" + Arrays.toString(Arrays.copyOf(data, end)) + "]]"; - } - -} diff --git a/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedIntArray.java b/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedIntArray.java deleted file mode 100644 index 1339266d..00000000 --- a/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedIntArray.java +++ /dev/null @@ -1,212 +0,0 @@ -package nu.marginalia.array.delegate; - -import nu.marginalia.array.ArrayRangeReference; -import nu.marginalia.array.IntArray; -import nu.marginalia.array.algo.SortingContext; -import nu.marginalia.array.buffer.IntQueryBuffer; -import nu.marginalia.array.functional.*; - -import java.io.IOException; -import java.nio.IntBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Path; - -public class ShiftedIntArray implements IntArray { - public final long shift; - public final long size; - - private final IntArray delegate; - - public ShiftedIntArray(long shift, IntArray delegate) { - this.shift = shift; - this.size = delegate.size() - shift; - this.delegate = delegate; - } - - public ShiftedIntArray(long start, long end, IntArray delegate) { - this.shift = start; - this.size = end - start; - this.delegate = delegate; - } - - @Override - public int get(long pos) { - return delegate.get(pos+shift); - } - - @Override - public void set(long pos, int value) { - delegate.set(pos+shift, value); - } - - @Override - public void set(long start, long end, IntBuffer buffer, int bufferStart) { - delegate.set(shift + start, shift + end, buffer, bufferStart); - } - - @Override - public void get(long start, long end, IntBuffer buffer, int bufferStart) { - delegate.get(shift + start, shift + end, buffer, bufferStart); - } - - @Override - public void get(long start, IntBuffer buffer) { - delegate.get(shift + start, buffer); - } - - @Override - public void get(long start, long end, int[] buffer) { - delegate.get(shift+start, shift+end, buffer); - } - - @Override - public int getAndIncrement(long pos) { - return delegate.getAndIncrement(shift + pos); - } - - @Override - public void fill(long start, long end, int val) { - delegate.fill(start + shift, end + shift, val); - } - - @Override - public long size() { - return size; - } - - @Override - public void write(Path file) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public ShiftedIntArray shifted(long offset) { - return new ShiftedIntArray(shift+offset, delegate); - } - - @Override - public ShiftedIntArray range(long start, long end) { - return new ShiftedIntArray(shift + start, shift+end, delegate); - } - - public ArrayRangeReference directRangeIfPossible(long start, long end) { - return delegate.directRangeIfPossible(shift + start, shift + end); - } - - public int[] toArray() { - int[] ret = new int[(int) size]; - for (int i = 0; i < size; i++) { - ret[i] = delegate.get(shift + i); - } - return ret; - } - - public boolean isSorted() { - return isSorted(0, size); - } - - public boolean isSorted(long start, long end) { - return delegate.isSorted(shift + start, shift + end); - } - - - public void sortLargeSpan(SortingContext ctx, long start, long end) throws IOException { - delegate.sortLargeSpan(ctx, start, end); - } - - - public long search(int key) { - if (size < 128) { - return linearSearch(key); - } - else { - return binarySearch(key); - } - } - - public long linearSearch(int key) { - return linearSearch(key, 0, size); - } - - public long binarySearch(int key) { - return binarySearch(key, 0, size); - } - - public long binarySearchUpperbound(int key) { - return binarySearchUpperBound(key, 0, size); - } - - public void retain(IntQueryBuffer buffer, long boundary) { - retain(buffer, boundary, 0, size); - } - - public void reject(IntQueryBuffer buffer, long boundary) { - reject(buffer, boundary, 0, size); - } - - @Override - public long linearSearch(int key, long fromIndex, long toIndex) { - return translateSearchResult(delegate.linearSearch(key, fromIndex + shift, toIndex+shift)); - } - - @Override - public long binarySearch(int key, long fromIndex, long toIndex) { - return translateSearchResult(delegate.binarySearch(key, fromIndex + shift, toIndex+shift)); - } - - @Override - public long binarySearchUpperBound(int key, long fromIndex, long toIndex) { - return translateSearchResult(delegate.binarySearchUpperBound(key, fromIndex + shift, toIndex+shift)); - } - - private long translateSearchResult(long ret) { - if (ret > 0) return ret - shift; - return ret + shift; - } - - @Override - public void retain(IntQueryBuffer buffer, long boundary, long searchStart, long searchEnd) { - delegate.retain(buffer, boundary, searchStart + shift, searchEnd + shift); - } - - @Override - public void reject(IntQueryBuffer buffer, long boundary, long searchStart, long searchEnd) { - delegate.reject(buffer, boundary, searchStart + shift, searchEnd + shift); - } - - @Override - public void forEach(long start, long end, LongIntConsumer consumer) { - delegate.forEach(start + shift, end+shift, (pos, old) -> consumer.accept(pos-shift, old)); - } - - @Override - public void transformEach(long start, long end, IntTransformer transformer) { - delegate.transformEach(start + shift, end+shift, (pos, old) -> transformer.transform(pos-shift, old)); - } - - @Override - public void transformEachIO(long start, long end, IntIOTransformer transformer) throws IOException { - delegate.transformEachIO(start + shift, end+shift, (pos, old) -> transformer.transform(pos-shift, old)); - } - - @Override - public int foldIO(int zero, long start, long end, IntBinaryIOOperation operator) throws IOException { - return delegate.foldIO(zero, start + shift, end+shift, operator); - } - - @Override - public int fold(int zero, long start, long end, IntBinaryOperation operator){ - return delegate.fold(zero, start + shift, end+shift, operator); - } - - @Override - public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException { - delegate.transferFrom(source, sourceStart, shift + arrayStart, shift + arrayEnd); - } - - @Override - public void force() { - delegate.force(); - } - -} diff --git a/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedLongArray.java b/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedLongArray.java deleted file mode 100644 index 7350b46f..00000000 --- a/code/libraries/array/java/nu/marginalia/array/delegate/ShiftedLongArray.java +++ /dev/null @@ -1,212 +0,0 @@ -package nu.marginalia.array.delegate; - -import nu.marginalia.array.ArrayRangeReference; -import nu.marginalia.array.LongArray; -import nu.marginalia.array.algo.LongArraySearch; -import nu.marginalia.array.buffer.LongQueryBuffer; -import nu.marginalia.array.functional.*; - -import java.io.IOException; -import java.nio.LongBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Path; - -public class ShiftedLongArray implements LongArray { - public final long shift; - public final long size; - private final LongArray delegate; - - public ShiftedLongArray(long shift, LongArray delegate) { - this.shift = shift; - this.size = delegate.size() - shift; - this.delegate = delegate; - } - - public ShiftedLongArray(long start, long end, LongArray delegate) { - this.shift = start; - this.size = end - start; - this.delegate = delegate; - } - - - @Override - public long get(long pos) { - return delegate.get(pos+shift); - } - - @Override - public void set(long pos, long value) { - delegate.set(pos+shift, value); - } - - @Override - public void set(long start, long end, LongBuffer buffer, int bufferStart) { - delegate.set(shift + start, shift + end, buffer, bufferStart); - } - - @Override - public void get(long start, long end, LongBuffer buffer, int bufferStart) { - delegate.get(shift + start, shift + end, buffer, bufferStart); - } - - @Override - public void close() { - delegate.close(); - } - - @Override - public void get(long start, LongBuffer buffer) { - delegate.get(shift + start, buffer); - } - - @Override - public void get(long start, long end, long[] buffer) { - delegate.get(shift+start, shift+end, buffer); - } - - @Override - public long getAndIncrement(long pos) { - return delegate.getAndIncrement(shift + pos); - } - - @Override - public void fill(long start, long end, long val) { - delegate.fill(start + shift, end + shift, val); - } - - @Override - public void quickSortNative(long start, long end) { - delegate.quickSortNative(start + shift, end + shift); - } - @Override - public void quickSortNative128(long start, long end) { - delegate.quickSortNative128(start, end); - } - - @Override - public long size() { - return size; - } - - @Override - public void write(Path file) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public ShiftedLongArray shifted(long offset) { - return new ShiftedLongArray(shift+offset, delegate); - } - - @Override - public ShiftedLongArray range(long start, long end) { - return new ShiftedLongArray(shift + start, shift+end, delegate); - } - - public long[] toArray() { - long[] ret = new long[(int) size]; - for (int i = 0; i < size; i++) { - ret[i] = delegate.get(shift + i); - } - return ret; - } - - public boolean isSorted() { - return isSorted(0, size); - } - - public boolean isSortedN(int sz) { - return isSortedN(sz, 0, size); - } - - public boolean isSorted(long start, long end) { - return delegate.isSorted(shift + start, shift + end); - } - - public void quickSort(long start, long end) { - delegate.quickSort(shift + start, shift + end); - } - - public void quickSortN(int sz, long start, long end) { - delegate.quickSortN(sz, shift + start, shift + end); - } - - public ArrayRangeReference directRangeIfPossible(long start, long end) { - return delegate.directRangeIfPossible(shift + start, shift + end); - } - - - public boolean isSortedN(int sz, long start, long end) { - return delegate.isSortedN(sz, shift + start, shift + end); - } - - - public long search(long key) { - return binarySearch(key, 0, size); - } - - public void retain(LongQueryBuffer buffer, long boundary) { - retain(buffer, boundary, 0, size); - } - public void reject(LongQueryBuffer buffer, long boundary) { - reject(buffer, boundary, 0, size); - } - - @Override - public long binarySearchN(int sz, long key, long fromIndex, long toIndex) { - return delegate.binarySearchN(sz, key, fromIndex + shift, toIndex+shift) - shift; - } - @Override - public long binarySearch(long key, long fromIndex, long toIndex) { - return delegate.binarySearch(key, fromIndex + shift, toIndex+shift) - shift; - } - - public void retain(LongQueryBuffer buffer, long boundary, long searchStart, long searchEnd) { - delegate.retain(buffer, boundary, searchStart + shift, searchEnd + shift); - } - public void retainN(LongQueryBuffer buffer, int sz, long boundary, long searchStart, long searchEnd) { - delegate.retainN(buffer, sz, boundary, searchStart + shift, searchEnd + shift); - } - public void reject(LongQueryBuffer buffer, long boundary, long searchStart, long searchEnd) { - delegate.reject(buffer, boundary, searchStart + shift, searchEnd + shift); - } - public void rejectN(LongQueryBuffer buffer, int sz, long boundary, long searchStart, long searchEnd) { - delegate.rejectN(buffer, sz, boundary, searchStart + shift, searchEnd + shift); - } - - @Override - public void forEach(long start, long end, LongLongConsumer consumer) { - delegate.forEach(start + shift, end+shift, (pos, old) -> consumer.accept(pos-shift, old)); - } - - @Override - public void transformEach(long start, long end, LongTransformer transformer) { - delegate.transformEach(start + shift, end+shift, (pos, old) -> transformer.transform(pos-shift, old)); - } - - @Override - public void transformEachIO(long start, long end, LongIOTransformer transformer) throws IOException { - delegate.transformEachIO(start + shift, end+shift, (pos, old) -> transformer.transform(pos-shift, old)); - } - - @Override - public long foldIO(long zero, long start, long end, LongBinaryIOOperation operator) throws IOException { - return delegate.foldIO(zero, start + shift, end+shift, operator); - } - - @Override - public long fold(long zero, long start, long end, LongBinaryOperation operator) { - return delegate.fold(zero, start + shift, end+shift, operator); - } - - @Override - public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException { - delegate.transferFrom(source, sourceStart, shift + arrayStart, shift + arrayEnd); - } - - @Override - public void force() { - delegate.force(); - } - -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCall.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCall.java deleted file mode 100644 index 90ed5ad8..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCall.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface AddressRangeCall { - void apply(T array, int start, int end); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCallIO.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCallIO.java deleted file mode 100644 index a52c1941..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeCallIO.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.array.functional; - -import java.io.IOException; - -public interface AddressRangeCallIO { - void apply(T array, int start, int end) throws IOException; -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeIntFunction.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeIntFunction.java deleted file mode 100644 index 30113292..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeIntFunction.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface AddressRangeIntFunction { - int apply(T array, int start, int end); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeLongFunction.java b/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeLongFunction.java deleted file mode 100644 index d97564be..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/AddressRangeLongFunction.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface AddressRangeLongFunction { - long apply(T array, int start, int end); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryIOOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryIOOperation.java deleted file mode 100644 index 5e3082ed..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryIOOperation.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.array.functional; - -import java.io.IOException; - -public interface IntBinaryIOOperation { - int apply(int left, int right) throws IOException; -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryOperation.java deleted file mode 100644 index ac6f6e8f..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/IntBinaryOperation.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface IntBinaryOperation { - int apply(int left, int right); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/IntIOTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/IntIOTransformer.java deleted file mode 100644 index 02675e27..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/IntIOTransformer.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.array.functional; - -import java.io.IOException; - -public interface IntIOTransformer { - int transform(long pos, int old) throws IOException; -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/IntTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/IntTransformer.java deleted file mode 100644 index 8001299c..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/IntTransformer.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface IntTransformer { - int transform(long pos, int old); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryIOOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryIOOperation.java deleted file mode 100644 index 8141d624..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryIOOperation.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.array.functional; - -import java.io.IOException; - -public interface LongBinaryIOOperation { - long apply(long left, long right) throws IOException; -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryOperation.java b/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryOperation.java deleted file mode 100644 index 4f1b1982..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/LongBinaryOperation.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface LongBinaryOperation { - long apply(long left, long right); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/LongIOTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongIOTransformer.java deleted file mode 100644 index 62fe490a..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/LongIOTransformer.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.array.functional; - -import java.io.IOException; - -public interface LongIOTransformer { - long transform(long pos, long old) throws IOException; -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/LongIntConsumer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongIntConsumer.java deleted file mode 100644 index a8a4ab2b..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/LongIntConsumer.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface LongIntConsumer { - void accept(long pos, int val); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/LongLongConsumer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongLongConsumer.java deleted file mode 100644 index 4707673d..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/LongLongConsumer.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface LongLongConsumer { - void accept(long pos, long val); -} diff --git a/code/libraries/array/java/nu/marginalia/array/functional/LongTransformer.java b/code/libraries/array/java/nu/marginalia/array/functional/LongTransformer.java deleted file mode 100644 index 8ab0ae7e..00000000 --- a/code/libraries/array/java/nu/marginalia/array/functional/LongTransformer.java +++ /dev/null @@ -1,5 +0,0 @@ -package nu.marginalia.array.functional; - -public interface LongTransformer { - long transform(long pos, long old); -} diff --git a/code/libraries/array/java/nu/marginalia/array/buffer/LongQueryBuffer.java b/code/libraries/array/java/nu/marginalia/array/page/LongQueryBuffer.java similarity index 99% rename from code/libraries/array/java/nu/marginalia/array/buffer/LongQueryBuffer.java rename to code/libraries/array/java/nu/marginalia/array/page/LongQueryBuffer.java index eaabb4a5..1a270af7 100644 --- a/code/libraries/array/java/nu/marginalia/array/buffer/LongQueryBuffer.java +++ b/code/libraries/array/java/nu/marginalia/array/page/LongQueryBuffer.java @@ -1,4 +1,4 @@ -package nu.marginalia.array.buffer; +package nu.marginalia.array.page; import nu.marginalia.array.LongArray; import nu.marginalia.array.LongArrayFactory; diff --git a/code/libraries/array/java/nu/marginalia/array/page/PartitionPage.java b/code/libraries/array/java/nu/marginalia/array/page/PartitionPage.java deleted file mode 100644 index 56afd660..00000000 --- a/code/libraries/array/java/nu/marginalia/array/page/PartitionPage.java +++ /dev/null @@ -1,22 +0,0 @@ -package nu.marginalia.array.page; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; - -public interface PartitionPage { - - default void write(FileChannel channel) throws IOException { - var byteBuffer = getByteBuffer(); - - byteBuffer.clear(); - - while (byteBuffer.position() < byteBuffer.limit()) { - channel.write(byteBuffer); - } - - byteBuffer.clear(); - } - - ByteBuffer getByteBuffer(); -} diff --git a/code/libraries/array/java/nu/marginalia/array/page/SegmentIntArray.java b/code/libraries/array/java/nu/marginalia/array/page/SegmentIntArray.java deleted file mode 100644 index 3aba6900..00000000 --- a/code/libraries/array/java/nu/marginalia/array/page/SegmentIntArray.java +++ /dev/null @@ -1,176 +0,0 @@ -package nu.marginalia.array.page; - -import nu.marginalia.array.ArrayRangeReference; -import nu.marginalia.array.IntArray; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.OpenOption; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -import static java.lang.foreign.ValueLayout.JAVA_INT; - -public class SegmentIntArray implements PartitionPage, IntArray { - - @Nullable - private final Arena arena; - private final MemorySegment segment; - private boolean closed; - - SegmentIntArray(MemorySegment segment, - @Nullable Arena arena) { - this.segment = segment; - this.arena = arena; - } - - public static SegmentIntArray onHeap(Arena arena, long size) { - return new SegmentIntArray(arena.allocate(WORD_SIZE*size, 8), arena); - } - - public static SegmentIntArray fromMmapReadOnly(Arena arena, Path file, long offset, long size) throws IOException { - return new SegmentIntArray( - mmapFile(arena, file, offset, size, FileChannel.MapMode.READ_ONLY, StandardOpenOption.READ), - arena); - } - - public static SegmentIntArray fromMmapReadWrite(Arena arena, Path file, long offset, long size) throws IOException { - - return new SegmentIntArray( - mmapFile(arena, file, offset, size, FileChannel.MapMode.READ_WRITE, - StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE), - arena); - } - - private static MemorySegment mmapFile(Arena arena, - Path file, - long offset, - long size, - FileChannel.MapMode mode, - OpenOption... openOptions) throws IOException - { - try (var channel = (FileChannel) Files.newByteChannel(file, openOptions)) { - - return channel.map(mode, - JAVA_INT.byteSize() * offset, - JAVA_INT.byteSize() * size, - arena); - } - catch (IOException ex) { - throw new IOException("Failed to map file " + file + " (" + offset + ":" + size + ")", ex); - } - } - - @Override - public IntArray range(long start, long end) { - return new SegmentIntArray( - segment.asSlice( - start * JAVA_INT.byteSize(), - (end-start) * JAVA_INT.byteSize()), - null); - } - - @Override - public IntArray shifted(long start) { - return new SegmentIntArray( - segment.asSlice(start * JAVA_INT.byteSize()), - null); - } - - @Override - public int get(long at) { - try { - return segment.getAtIndex(JAVA_INT, at); - } - catch (IndexOutOfBoundsException ex) { - throw new IndexOutOfBoundsException("@" + at + "(" + 0 + ":" + segment.byteSize()/8 + ")"); - } - } - - @Override - public void get(long start, long end, int[] buffer) { - for (int i = 0; i < end - start; i++) { - buffer[i] = segment.getAtIndex(JAVA_INT, start + i); - } - } - - @Override - public void set(long at, int val) { - segment.setAtIndex(JAVA_INT, at, val); - } - - @Override - public void set(long start, long end, IntBuffer buffer, int bufferStart) { - for (int i = 0; i < end - start; i++) { - set(start + i, buffer.get(bufferStart + i)); - } - } - - @Override - public synchronized void close() { - if (arena != null && !closed) { - arena.close(); - } - closed = true; - } - - @Override - public long size() { - return segment.byteSize() / JAVA_INT.byteSize(); - } - - @Override - public ByteBuffer getByteBuffer() { - return segment.asByteBuffer(); - } - - @Override - public void write(Path filename) throws IOException { - try (var arena = Arena.ofConfined()) { - var destSegment = SegmentIntArray.fromMmapReadWrite(arena, filename, 0, segment.byteSize()); - - destSegment.segment.copyFrom(segment); - destSegment.force(); - } - } - - @Override - public void force() { - if (segment.isMapped()) { - segment.force(); - } - } - - public ArrayRangeReference directRangeIfPossible(long start, long end) { - return new ArrayRangeReference<>(this, start, end); - } - - @Override - public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException { - - final int stride = 1024*1204*128; // Copy 1 GB at a time 'cause byte buffers are 'a byte buffering - - long ss = sourceStart; - for (long as = arrayStart; as < arrayEnd; as += stride, ss += stride) { - long ae = Math.min(as + stride, arrayEnd); - - long index = as * JAVA_INT.byteSize(); - long length = (ae - as) * JAVA_INT.byteSize(); - - var bufferSlice = segment.asSlice(index, length).asByteBuffer(); - - long startPos = ss * JAVA_INT.byteSize(); - while (bufferSlice.position() < bufferSlice.capacity()) { - source.read(bufferSlice, startPos + bufferSlice.position()); - } - } - - } - -} diff --git a/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java b/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java index 034752ab..ac420de9 100644 --- a/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java +++ b/code/libraries/array/java/nu/marginalia/array/page/SegmentLongArray.java @@ -1,14 +1,11 @@ package nu.marginalia.array.page; -import nu.marginalia.NativeAlgos; -import nu.marginalia.array.ArrayRangeReference; import nu.marginalia.array.LongArray; import javax.annotation.Nullable; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.nio.LongBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; @@ -18,7 +15,8 @@ import java.nio.file.StandardOpenOption; import static java.lang.foreign.ValueLayout.JAVA_LONG; -public class SegmentLongArray implements PartitionPage, LongArray { +@SuppressWarnings("preview") +public class SegmentLongArray implements LongArray { @Nullable private final Arena arena; @@ -126,20 +124,6 @@ public class SegmentLongArray implements PartitionPage, LongArray { return segment.byteSize() / JAVA_LONG.byteSize(); } - @Override - public void quickSortNative(long start, long end) { - NativeAlgos.sort(segment, start, end); - } - @Override - public void quickSortNative128(long start, long end) { - NativeAlgos.sort128(segment, start, end); - } - - @Override - public ByteBuffer getByteBuffer() { - return segment.asByteBuffer(); - } - @Override public void write(Path filename) throws IOException { try (var arena = Arena.ofConfined()) { @@ -158,10 +142,6 @@ public class SegmentLongArray implements PartitionPage, LongArray { } - public ArrayRangeReference directRangeIfPossible(long start, long end) { - return new ArrayRangeReference<>(this, start, end); - } - @Override public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException { @@ -188,4 +168,8 @@ public class SegmentLongArray implements PartitionPage, LongArray { } + @Override + public MemorySegment getMemorySegment() { + return segment; + } } diff --git a/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java b/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java index 7d86e56e..04ea42d4 100644 --- a/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java +++ b/code/libraries/array/java/nu/marginalia/array/page/UnsafeLongArray.java @@ -1,7 +1,5 @@ package nu.marginalia.array.page; -import nu.marginalia.NativeAlgos; -import nu.marginalia.array.ArrayRangeReference; import nu.marginalia.array.LongArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,7 +10,6 @@ import javax.annotation.Nullable; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteBuffer; import java.nio.LongBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; @@ -23,7 +20,9 @@ import static java.lang.foreign.ValueLayout.JAVA_LONG; /** Variant of SegmentLongArray that uses Unsafe to access the memory. * */ -public class UnsafeLongArray implements PartitionPage, LongArray { + +@SuppressWarnings("preview") +public class UnsafeLongArray implements LongArray { private static final Unsafe unsafe = UnsafeProvider.getUnsafe(); private static final Logger logger = LoggerFactory.getLogger(UnsafeLongArray.class); @@ -114,6 +113,11 @@ public class UnsafeLongArray implements PartitionPage, LongArray { unsafe.putLong(segment.address() + at * JAVA_LONG.byteSize(), val); } + @Override + public MemorySegment getMemorySegment() { + return segment; + } + @Override public void set(long start, long end, LongBuffer buffer, int bufferStart) { for (int i = 0; i < end - start; i++) { @@ -143,11 +147,6 @@ public class UnsafeLongArray implements PartitionPage, LongArray { return segment.byteSize() / JAVA_LONG.byteSize(); } - @Override - public ByteBuffer getByteBuffer() { - return segment.asByteBuffer(); - } - @Override public void write(Path filename) throws IOException { try (var arena = Arena.ofConfined()) { @@ -172,10 +171,6 @@ public class UnsafeLongArray implements PartitionPage, LongArray { } } - public ArrayRangeReference directRangeIfPossible(long start, long end) { - return new ArrayRangeReference<>(this, start, end); - } - public void chanelChannelTransfer(FileChannel source, long sourceStartL, long arrayStartL, @@ -274,14 +269,4 @@ public class UnsafeLongArray implements PartitionPage, LongArray { } } - @Override - public void quickSortNative(long start, long end) { - NativeAlgos.sort(segment, start, end); - } - - @Override - public void quickSortNative128(long start, long end) { - NativeAlgos.sort128(segment, start, end); - } - } diff --git a/code/libraries/array/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java b/code/libraries/array/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java deleted file mode 100644 index e705d0a3..00000000 --- a/code/libraries/array/java/nu/marginalia/array/scheme/ArrayPartitioningScheme.java +++ /dev/null @@ -1,51 +0,0 @@ -package nu.marginalia.array.scheme; - -public interface ArrayPartitioningScheme { - - static ArrayPartitioningScheme forPartitionSize(int size) { - if (Integer.highestOneBit(size) == size) { - return new PowerOf2PartitioningScheme(size); - } - else { - return new SequentialPartitioningScheme(size); - } - } - static int getRequiredPartitions(long cardinality, int partitionSize) { - return (int) (cardinality / partitionSize + Long.signum(cardinality % partitionSize)); - } - - - int getPartitions(long cardinality); - - int getPage(long at); - - boolean isSamePage(long a, long b); - - /** Get the page offset corresponding to at */ - int getOffset(long at); - - /** Variant of getOffset that doesn't wrap around the page boundary, necessary when - * translating an exclusive end offset that getOffset(...) will translate to 0 and consider - * part of the next page. - * - * It is also necessary to consider the start offset to determine when the end offset - * - */ - default int getEndOffset(long start, long end) { - if (end == 0 || end <= start) - return getOffset(end); - - return 1 + getOffset(end - 1); - } - - /** Get the end of the buffer containing at, or endTotal, whichever is smaller - */ - long getPageEnd(long at, long endTotal); - - /** - * toRealIndex(getBuffer(val), getOffset(val)) = val - */ - long toRealIndex(int buffer, int offset); - - int getRequiredPageSize(int buffer, long cardinality); -} diff --git a/code/libraries/array/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java b/code/libraries/array/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java deleted file mode 100644 index 98dfac8f..00000000 --- a/code/libraries/array/java/nu/marginalia/array/scheme/PowerOf2PartitioningScheme.java +++ /dev/null @@ -1,60 +0,0 @@ -package nu.marginalia.array.scheme; - -public class PowerOf2PartitioningScheme implements ArrayPartitioningScheme { - final int partitionSize; - final long offsetMask; - final long bufferMask; - final int pageShift; - - public PowerOf2PartitioningScheme(int partitionSize) { - assert partitionSize == Integer.highestOneBit(partitionSize); - - this.partitionSize = partitionSize; - - offsetMask = partitionSize - 1; - bufferMask = ~offsetMask; - pageShift = Integer.numberOfTrailingZeros(partitionSize); - } - - @Override - public int getPartitions(long cardinality) { - return ArrayPartitioningScheme.getRequiredPartitions(cardinality, partitionSize); - } - - @Override - public int getPage(long at) { // very hot code - return (int) (at >>> pageShift); - } - - @Override - public int getOffset(long at) { // very hot code - return (int) (at & offsetMask); - } - - @Override - public boolean isSamePage(long a, long b) { // hot code - return 0 == ((a ^ b) & bufferMask); - } - - @Override - public long getPageEnd(long at, long endTotal) { - return Math.min(endTotal, partitionSize * (1L + getPage(at))); - } - - @Override - public long toRealIndex(int buffer, int offset) { - return offset + (long) buffer * partitionSize; - } - - @Override - public int getRequiredPageSize(int buffer, long cardinality) { - - if ((long) (1 + buffer) * partitionSize <= cardinality) { - return partitionSize; - } - - return (int) (cardinality % partitionSize); - } - - -} diff --git a/code/libraries/array/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java b/code/libraries/array/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java deleted file mode 100644 index 7bd3f0a7..00000000 --- a/code/libraries/array/java/nu/marginalia/array/scheme/SequentialPartitioningScheme.java +++ /dev/null @@ -1,56 +0,0 @@ -package nu.marginalia.array.scheme; - -public class SequentialPartitioningScheme implements ArrayPartitioningScheme { - - final int partitionSize; - - public SequentialPartitioningScheme(int partitionSize) { - this.partitionSize = partitionSize; - } - - public static int getRequiredPartitions(long cardinality, int partitionSize) { - return (int) (cardinality / partitionSize + Long.signum(cardinality % partitionSize)); - } - - @Override - public int getPartitions(long cardinality) { - return getRequiredPartitions(cardinality, partitionSize); - } - - @Override - public int getPage(long at) { - return (int) (at / partitionSize); - } - - public long getPageEnd(long at, long endTotal) { - return Math.min(endTotal, partitionSize * (1L + getPage(at))); - } - - - @Override - public boolean isSamePage(long a, long b) { - return (int) (a / partitionSize) == (int)(b/partitionSize); - } - - @Override - public int getOffset(long at) { - return (int) (at % partitionSize); - } - - public long toRealIndex(int buffer, int offset) { - return offset + (long) buffer * partitionSize; - } - - - - @Override - public int getRequiredPageSize(int buffer, long cardinality) { - - if ((long) (1 + buffer) * partitionSize <= cardinality) { - return partitionSize; - } - return (int) (cardinality % partitionSize); - } - - -} diff --git a/code/libraries/array/readme.md b/code/libraries/array/readme.md index 9848792b..f656b3e1 100644 --- a/code/libraries/array/readme.md +++ b/code/libraries/array/readme.md @@ -36,11 +36,9 @@ try (var array = LongArrayFactory.mmapForWritingConfined(Path.of("/tmp/test"), 1 ## Query Buffers -The classes [IntQueryBuffer](java/nu/marginalia/array/buffer/IntQueryBuffer.java) -and [LongQueryBuffer](java/nu/marginalia/array/buffer/LongQueryBuffer.java) are used -heavily in the search engine's query processing. +The class and [LongQueryBuffer](java/nu/marginalia/array/buffer/LongQueryBuffer.java) is used heavily in the search engine's query processing. -They are dual-pointer buffers that offer tools for filtering data. +It is a dual-pointer buffer that offers tools for filtering data. ```java LongQueryBuffer buffer = new LongQueryBuffer(1000); @@ -78,8 +76,7 @@ buffer.finalizeFiltering(); ``` -Especially noteworthy are the operations `retain()` and `reject()` in -[IntArraySearch](java/nu/marginalia/array/algo/IntArraySearch.java) and [LongArraySearch](java/nu/marginalia/array/algo/LongArraySearch.java). +Especially noteworthy are the operations `retain()` and `reject()` in [LongArraySearch](java/nu/marginalia/array/algo/LongArraySearch.java). They keep or remove all items in the buffer that exist in the referenced range of the array, which must be sorted. diff --git a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SearchBenchmark.java b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SearchBenchmark.java index 898a2f33..723443c1 100644 --- a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SearchBenchmark.java +++ b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SearchBenchmark.java @@ -10,14 +10,14 @@ import java.util.Random; public class SearchBenchmark { @State(Scope.Benchmark) - public static class SortState { + public static class SearchState { - public SortState() + public SearchState() { msArray.transformEach(0, size, (pos,old) -> ~pos); usArray.transformEach(0, size, (pos,old) -> ~pos); - msArray.quickSortJava(0, size); - usArray.quickSortJava(0, size); + msArray.sort(0, size); + usArray.sort(0, size); keys = new long[1000]; Random r = new Random(); for (int i = 0; i < 1000; i++) { @@ -36,27 +36,12 @@ public class SearchBenchmark { @Warmup(iterations = 5) @Benchmark @BenchmarkMode(Mode.Throughput) - public long msSort64(SortState state) { + public long msSearch64(SearchState state) { var array = state.usArray; long ret = 0; for (var key : state.keys) { - ret += array.binarySearchNJava(2, key, 0, array.size()); - } - - return ret; - } - - @Fork(value = 3, warmups = 5) - @Warmup(iterations = 5) - @Benchmark - @BenchmarkMode(Mode.Throughput) - public long msSort64_2(SortState state) { - var array = state.usArray; - - long ret = 0; - for (var key : state.keys) { - ret += array.binarySearchNJava2(2, key, 0, array.size()); + ret += array.binarySearch(key, 0, array.size()); } return ret; @@ -66,12 +51,12 @@ public class SearchBenchmark { @Warmup(iterations = 1) @Benchmark @BenchmarkMode(Mode.Throughput) - public long msSort128(SortState state) { + public long msSearch128(SearchState state) { var array = state.msArray; long ret = 0; for (var key : state.keys) { - ret += array.binarySearchNJava(2, 0, array.size(), key); + ret += array.binarySearchN(2, 0, array.size(), key); } return ret; @@ -81,12 +66,12 @@ public class SearchBenchmark { @Warmup(iterations = 1) @Benchmark @BenchmarkMode(Mode.Throughput) - public long usSort64(SortState state) { + public long usSearch64(SearchState state) { var array = state.usArray; long ret = 0; for (var key : state.keys) { - ret += array.binarySearchUpperBoundJava(0, array.size(), key); + ret += array.binarySearch(0, array.size(), key); } return ret; @@ -96,12 +81,12 @@ public class SearchBenchmark { @Warmup(iterations = 1) @Benchmark @BenchmarkMode(Mode.Throughput) - public long usSort128(SortState state) { + public long usSearch128(SearchState state) { var array = state.usArray; long ret = 0; for (var key : state.keys) { - ret += array.binarySearchNJava(2, 0, array.size(), key); + ret += array.binarySearchN(2, 0, array.size(), key); } return ret; diff --git a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SortBenchmark.java b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SortBenchmark.java index 70c8c591..603339b5 100644 --- a/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SortBenchmark.java +++ b/code/libraries/array/src/jmh/java/nu/marginalia/array/page/SortBenchmark.java @@ -1,6 +1,8 @@ package nu.marginalia.array.page; +import nu.marginalia.NativeAlgos; import nu.marginalia.array.LongArray; +import nu.marginalia.array.algo.LongArraySort; import org.openjdk.jmh.annotations.*; import java.lang.foreign.Arena; @@ -30,31 +32,7 @@ public class SortBenchmark { public LongArray msSort64(BenchState state) { var array = state.msArray; - array.quickSortJavaN(2, 0, array.size()); - - return array; - } - - @Fork(value = 5, warmups = 5) - @Warmup(iterations = 1) - @Benchmark - @BenchmarkMode(Mode.Throughput) - public LongArray msSort128(BenchState state) { - var array = state.msArray; - - array.quickSortJavaN(2, 0, array.size()); - - return array; - } - - @Fork(value = 5, warmups = 5) - @Warmup(iterations = 1) - @Benchmark - @BenchmarkMode(Mode.Throughput) - public LongArray usSort128(BenchState state) { - var array = state.usArray; - - array.quickSortJavaN(2, 0, array.size()); + LongArraySort.quickSortJava(array, 0, array.size()); return array; } @@ -66,7 +44,55 @@ public class SortBenchmark { public LongArray usSort64(BenchState state) { var array = state.usArray; - array.quickSortJavaN(2, 0, array.size()); + LongArraySort.quickSortJava(array,0, array.size()); + + return array; + } + + @Fork(value = 5, warmups = 5) + @Warmup(iterations = 1) + @Benchmark + @BenchmarkMode(Mode.Throughput) + public LongArray msSort128(BenchState state) { + var array = state.msArray; + + LongArraySort.quickSortJavaN(array,2, 0, array.size()); + + return array; + } + + @Fork(value = 5, warmups = 5) + @Warmup(iterations = 1) + @Benchmark + @BenchmarkMode(Mode.Throughput) + public LongArray usSort128(BenchState state) { + var array = state.usArray; + + LongArraySort.quickSortJavaN(array,2, 0, array.size()); + + return array; + } + + @Fork(value = 5, warmups = 5) + @Warmup(iterations = 1) + @Benchmark + @BenchmarkMode(Mode.Throughput) + public LongArray msSort128_2(BenchState state) { + var array = state.msArray; + + LongArraySort.quickSortJava2(array, 0, array.size()); + + return array; + } + + @Fork(value = 5, warmups = 5) + @Warmup(iterations = 1) + @Benchmark + @BenchmarkMode(Mode.Throughput) + public LongArray usSort128_2(BenchState state) { + var array = state.usArray; + + LongArraySort.quickSortJava2(array,0, array.size()); return array; } @@ -82,7 +108,7 @@ public class SortBenchmark { var array = state.usArray; // realistically doesn't matter - array.quickSortNative128(0, array.size()); + NativeAlgos.sort128(array.getMemorySegment(), 0, array.size()); return array; } @@ -95,7 +121,7 @@ public class SortBenchmark { var array = state.usArray; // realistically doesn't matter - array.quickSortNative(0, array.size()); + NativeAlgos.sort(array.getMemorySegment(), 0, array.size()); return array; } diff --git a/code/libraries/array/test/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java b/code/libraries/array/test/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java deleted file mode 100644 index cc5b8cd3..00000000 --- a/code/libraries/array/test/nu/marginalia/array/IntLowBitPartitioningSchemeTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package nu.marginalia.array; - -import nu.marginalia.array.scheme.SequentialPartitioningScheme; -import org.junit.jupiter.api.Test; - -class IntLowBitPartitioningSchemeTest { - - @Test - public void testLBPT() { - var p = new SequentialPartitioningScheme(18); - - System.out.println(p.getRequiredPageSize(0, 51)); - System.out.println(p.getRequiredPageSize(1, 51)); - System.out.println(p.getRequiredPageSize(2, 51)); - System.out.println(p.getRequiredPageSize(3, 51)); - - for (int i = 0; i < 100; i++) { - System.out.println(p.getPage(i) + ":" + p.getOffset(i)); - } - } -} \ No newline at end of file diff --git a/code/libraries/array/test/nu/marginalia/array/algo/IntArraySearchTest.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArraySearchTest.java deleted file mode 100644 index 9c1680a9..00000000 --- a/code/libraries/array/test/nu/marginalia/array/algo/IntArraySearchTest.java +++ /dev/null @@ -1,132 +0,0 @@ -package nu.marginalia.array.algo; - -import nu.marginalia.array.IntArray; -import nu.marginalia.array.buffer.IntQueryBuffer; -import nu.marginalia.array.page.SegmentIntArray; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.lang.foreign.Arena; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class IntArraySearchTest { - - IntArray basicArray = IntArray.allocate(1024); - IntArray pagingArray = SegmentIntArray.onHeap(Arena.global(), 1024); - - IntArray shiftedArray = IntArray.allocate(1054).range(30, 1054); - - @BeforeEach - public void setUp() { - for (int i = 0; i < basicArray.size(); i++) { - basicArray.set(i, 3*i); - pagingArray.set(i, 3*i); - shiftedArray.set(i, 3*i); - } - } - - @Test - void linearSearch() { - linearSearchTester(basicArray); - linearSearchTester(pagingArray); - linearSearchTester(shiftedArray); - } - - @Test - void binarySearch() { - binarySearchTester(basicArray); - binarySearchTester(pagingArray); - binarySearchTester(shiftedArray); - } - - @Test - void binarySearchUpperbound() { - binarySearchUpperBoundTester(basicArray); - binarySearchUpperBoundTester(pagingArray); - binarySearchUpperBoundTester(shiftedArray); - } - - void linearSearchTester(IntArray array) { - for (int i = 0; i < array.size() * 3; i++) { - long ret = array.linearSearch(i, 0, array.size()); - - if ((i % 3) == 0) { - assertTrue(ret >= 0); - assertEquals(i, array.get(ret)); - } - else { - long higher = LongArraySearch.decodeSearchMiss(1, ret); - if (i > 0 && higher < array.size()) { - assertTrue(array.get(higher) < i); - } - } - } - } - - void binarySearchTester(IntArray array) { - for (int i = 0; i < array.size() * 3; i++) { - long ret = array.binarySearch(i, 0, array.size()); - - if ((i % 3) == 0) { - assertTrue(ret >= 0); - assertEquals(i, array.get(ret)); - } - else { - long higher = LongArraySearch.decodeSearchMiss(1, ret); - if (i > 0 && higher+1 < array.size()) { - assertTrue(array.get(higher) < i); - } - } - } - } - - void binarySearchUpperBoundTester(IntArray array) { - for (int i = 0; i < array.size() * 3; i++) { - long ret = array.binarySearchUpperBound(i, 0, array.size()); - - if ((i % 3) == 0) { - assertTrue(ret >= 0); - assertEquals(i, array.get(ret)); - } - else { - if (i > 0 && ret > 0 && ret < array.size()) { - assertTrue(array.get(ret-1) < i); - } - } - } - } - - @Test - void retain() { - int[] vals = new int[128]; - for (int i = 0; i < vals.length; i++) { vals[i] = i; } - var buffer = new IntQueryBuffer(vals, 128); - - basicArray.retain(buffer, 128, 0, basicArray.size()); - buffer.finalizeFiltering(); - - assertEquals(43, buffer.size()); - for (int i = 0; i < 43; i++) { - assertEquals(buffer.data[i], i*3); - } - } - - @Test - void reject() { - int[] vals = new int[128]; - for (int i = 0; i < vals.length; i++) { vals[i] = i; } - var buffer = new IntQueryBuffer(vals, 128); - - basicArray.reject(buffer, 128, 0, basicArray.size()); - buffer.finalizeFiltering(); - - assertEquals(128-43, buffer.size()); - int j = 0; - for (int i = 0; i < 43; i++) { - if (++j % 3 == 0) j++; - assertEquals(buffer.data[i], j); - } - } -} \ No newline at end of file diff --git a/code/libraries/array/test/nu/marginalia/array/algo/IntArraySortTest.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArraySortTest.java deleted file mode 100644 index 8dc16dab..00000000 --- a/code/libraries/array/test/nu/marginalia/array/algo/IntArraySortTest.java +++ /dev/null @@ -1,149 +0,0 @@ -package nu.marginalia.array.algo; - -import nu.marginalia.array.IntArray; -import nu.marginalia.util.test.TestUtil; -import org.apache.commons.lang3.ArrayUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Random; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@Tag("slow") -class IntArraySortTest { - - IntArray basic; - IntArray paged; - IntArray shifted; - - final int size = 1026; - - @BeforeEach - public void setUp() { - basic = IntArray.allocate(size); - paged = IntArray.allocate(size); - shifted = IntArray.allocate(size+30).shifted(30); - - var random = new Random(); - int[] values = new int[size]; - for (int i = 0; i < size; i++) { - values[i] = random.nextInt(0, 1000); - } - - basic.transformEach(0, size, (i, old) -> values[(int) i]); - paged.transformEach(0, size, (i, old) -> values[(int) i]); - shifted.transformEach(0, size, (i, old) -> values[(int) i]); - } - - interface SortOperation { - void sort(IntArray array, long start, long end) throws IOException; - } - - @Test - public void quickSortStressTest() throws IOException { - IntArray array = IntArray.allocate(65536); - sortAlgorithmTester(array, IntArraySort::quickSort); - } - - - @Test - public void insertionSortStressTest() throws IOException { - IntArray array = IntArray.allocate(8192); - sortAlgorithmTester(array, IntArraySort::insertionSort); - } - - @Test - public void mergeSortStressTest() throws IOException { - IntArray array = IntArray.allocate(65536); - Path tempDir = Files.createTempDirectory(getClass().getSimpleName()); - sortAlgorithmTester(array, (a, s, e) -> a.mergeSort(s, e, tempDir)); - TestUtil.clearTempDir(tempDir); - } - - void sortAlgorithmTester(IntArray array, SortOperation operation) throws IOException { - - int[] values = new int[(int) array.size()]; - for (int i = 0; i < values.length; i++) { - values[i] = i; - } - - ArrayUtils.shuffle(values); - - int sentinelA = 0xFEEDBEEF; - int sentinelB = 0xB000B000; - - int start = 6; - for (int end = start + 1; end < values.length - 1; end+=97) { - - // Use sentinel values to catch if the sort algorithm overwrites end values - array.set(start - 1, sentinelA); - array.set(end, sentinelB); - - long orderInvariantChecksum = 0; - for (long i = 0; i < end - start; i++) { - array.set(start + i, values[start + (int)i]); - - // Try to checksum the contents to catch bugs where the result is sorted - // but a value has been duplicated, overwriting another - orderInvariantChecksum ^= values[start + (int)i]; - } - - operation.sort(array, start, end); - - assertTrue(array.isSorted(start, end), "Array wasn't sorted"); - - assertEquals(sentinelA, array.get(start - 1), "Start position sentinel overwritten"); - assertEquals(sentinelB, array.get(end), "End position sentinel overwritten"); - - long actualChecksum = 0; - for (long i = start; i < end; i++) { - actualChecksum ^= array.get(i); - } - - assertEquals(orderInvariantChecksum, actualChecksum, "Checksum validation failed"); - } - - } - - @Test - void insertionSort() { - basic.insertionSort(0, size); - assertTrue(basic.isSorted(0, 128)); - - paged.insertionSort(0, size); - assertTrue(paged.isSorted(0, 128)); - - shifted.insertionSort(0, size); - assertTrue(shifted.isSorted(0, 128)); - } - - @Test - void quickSort() { - basic.quickSort(0, size); - assertTrue(basic.isSorted(0, size)); - - paged.quickSort(0, size); - assertTrue(paged.isSorted(0, size)); - - shifted.quickSort(0, size); - assertTrue(shifted.isSorted(0, 128)); - } - - @Test - void mergeSort() throws IOException { - basic.mergeSort(0, size, Path.of("/tmp")); - assertTrue(basic.isSorted(0, size)); - - paged.mergeSort(0, size, Path.of("/tmp")); - assertTrue(paged.isSorted(0, size)); - - shifted.mergeSort(0, size, Path.of("/tmp")); - assertTrue(shifted.isSorted(0, 128)); - } -} \ No newline at end of file diff --git a/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformations2Test.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformations2Test.java deleted file mode 100644 index b44ae07e..00000000 --- a/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformations2Test.java +++ /dev/null @@ -1,94 +0,0 @@ -package nu.marginalia.array.algo; - -import nu.marginalia.array.IntArray; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.ArrayList; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class IntArrayTransformations2Test { - IntArray basic; - IntArray paged; - IntArray shifted; - - final int size = 1026; - - @BeforeEach - public void setUp() { - basic = IntArray.allocate(size); - paged = IntArray.allocate(size); - shifted = IntArray.allocate(size+30).shifted(30); - - int[] vals = new int[size]; - for (int i = 0; i < vals.length; i++) { - vals[i] = i+10; - } - basic.set(0, vals); - paged.set(0, vals); - shifted.set(0, vals); - } - @Test - void forEach() { - foreachTester(basic); - foreachTester(paged); - foreachTester(shifted); - } - @Test - void transformEach() { - transformTester(basic); - transformTester(paged); - transformTester(shifted); - } - - @Test - void transformEachIO() throws IOException { - transformTesterIO(basic); - transformTesterIO(paged); - transformTesterIO(shifted); - } - - private void transformTester(IntArray array) { - ArrayList offsets = new ArrayList<>(); - - array.transformEach(0, size, (i, val) -> { - assertEquals(i+10, val); - offsets.add(i); - return -val; - }); - for (int i = 0; i < size; i++) { - assertEquals(-(i+10), array.get(i)); - } - } - - private void transformTesterIO(IntArray array) throws IOException { - ArrayList offsets = new ArrayList<>(); - - array.transformEachIO(0, size, (i, val) -> { - assertEquals(i+10, val); - offsets.add(i); - return -val; - }); - - for (int i = 0; i < size; i++) { - assertEquals(-(i+10), array.get(i)); - } - - for (int i = 0; i < size; i++) { - assertEquals(offsets.get(i), i); - } - } - - private void foreachTester(IntArray array) { - ArrayList offsets = new ArrayList<>(); - array.forEach(0, size, (i, val) -> { - assertEquals(i+10, val); - offsets.add(i); - }); - for (int i = 0; i < size; i++) { - assertEquals(offsets.get(i), i); - } - } -} \ No newline at end of file diff --git a/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformationsTest.java b/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformationsTest.java deleted file mode 100644 index 37954f27..00000000 --- a/code/libraries/array/test/nu/marginalia/array/algo/IntArrayTransformationsTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package nu.marginalia.array.algo; - -import nu.marginalia.array.IntArray; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class IntArrayTransformationsTest { - IntArray basic; - IntArray paged; - IntArray shifted; - - final int size = 1026; - - @BeforeEach - public void setUp() { - basic = IntArray.allocate(size); - paged = IntArray.allocate(size); - shifted = IntArray.allocate(size+30).shifted(30); - - for (int i = 0; i < basic.size(); i++) { - basic.set(i, 3*i); - paged.set(i, 3*i); - shifted.set(i, 3*i); - } - } - - @Test - void transformEach() { - transformTester(basic); - transformTester(paged); - transformTester(shifted); - } - - @Test - void transformEachIO() throws IOException { - transformTesterIO(basic); - transformTesterIO(paged); - transformTesterIO(shifted); - } - - @Test - void foldIO() throws IOException { - assertEquals(3*(5+6+7+8+9), basic.foldIO(0, 5, 10, Integer::sum)); - assertEquals(3*(5+6+7+8+9), paged.foldIO(0, 5, 10, Integer::sum)); - assertEquals(3*(5+6+7+8+9), shifted.foldIO(0, 5, 10, Integer::sum)); - } - - @Test - void fold() { - assertEquals(3*(5+6+7+8+9), basic.fold(0, 5, 10, Integer::sum)); - assertEquals(3*(5+6+7+8+9), paged.fold(0, 5, 10, Integer::sum)); - assertEquals(3*(5+6+7+8+9), shifted.fold(0, 5, 10, Integer::sum)); - } - - private void transformTester(IntArray array) { - array.transformEach(5, 15, (i, o) -> (int) (i - o)); - for (int i = 0; i < 5; i++) { - assertEquals(3*i, array.get(i)); - } - for (int i = 5; i < 15; i++) { - assertEquals(-2*i, array.get(i)); - } - for (int i = 15; i < 20; i++) { - assertEquals(3*i, array.get(i)); - } - } - - private void transformTesterIO(IntArray array) throws IOException { - array.transformEachIO(5, 15, (i, o) -> (int) (i - o)); - for (int i = 0; i < 5; i++) { - assertEquals(3*i, array.get(i)); - } - for (int i = 5; i < 15; i++) { - assertEquals(-2*i, array.get(i)); - } - for (int i = 15; i < 20; i++) { - assertEquals(3*i, array.get(i)); - } - } -} \ No newline at end of file diff --git a/code/libraries/array/test/nu/marginalia/array/algo/LongArraySearchTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySearchTest.java index f551fa19..2d3d2db9 100644 --- a/code/libraries/array/test/nu/marginalia/array/algo/LongArraySearchTest.java +++ b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySearchTest.java @@ -2,7 +2,7 @@ package nu.marginalia.array.algo; import nu.marginalia.array.LongArray; import nu.marginalia.array.LongArrayFactory; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortNTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortNTest.java index 8094d956..dfbf555e 100644 --- a/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortNTest.java +++ b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortNTest.java @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class LongArraySortNTest { LongArray basic; - LongArray paged; LongArray shifted; LongArray segment; @@ -33,7 +32,6 @@ class LongArraySortNTest { @BeforeEach public void setUp() { basic = LongArray.allocate(size); - paged = LongArray.allocate(size); shifted = LongArray.allocate(size+30).shifted(30); segment = LongArrayFactory.onHeapShared(size + 30).shifted(30); @@ -47,7 +45,6 @@ class LongArraySortNTest { } basic.set(0, values); - paged.set(0, values); shifted.set(0, values); segment.set(0, values); @@ -62,7 +59,7 @@ class LongArraySortNTest { @Test public void quickSortStressTest() throws IOException { LongArray array = LongArray.allocate(65536); - sortAlgorithmTester(array, LongArraySort::quickSort); + sortAlgorithmTester(array, LongArraySort::sort); } @@ -132,20 +129,16 @@ class LongArraySortNTest { @Test void insertionSortN() { - basic.insertionSortN(2, 0, size); + LongArraySort.insertionSortN(basic, 2, 0, size); assertTrue(basic.isSortedN(2, 0, size)); - paged.insertionSortN(2, 0, size); - assertTrue(paged.isSortedN(2, 0, size)); - - shifted.insertionSortN(2, 0, size); + LongArraySort.insertionSortN(shifted, 2, 0, size); assertTrue(shifted.isSortedN(2, 0, size)); - segment.insertionSortN(2, 0, size); + LongArraySort.insertionSortN(segment, 2, 0, size); assertTrue(segment.isSortedN(2, 0, size)); compare(basic, dataAsPairs); - compare(paged, dataAsPairs); compare(shifted, dataAsPairs); compare(segment, dataAsPairs); } @@ -155,9 +148,6 @@ class LongArraySortNTest { basic.quickSortN(2, 0, size); assertTrue(basic.isSortedN(2, 0, size)); - paged.quickSortN(2, 0, size); - assertTrue(paged.isSortedN(2, 0, size)); - shifted.quickSortN(2, 0, size); assertTrue(shifted.isSortedN(2, 0, size)); @@ -165,7 +155,6 @@ class LongArraySortNTest { assertTrue(segment.isSortedN(2, 0, size)); compare(basic, dataAsPairs); - compare(paged, dataAsPairs); compare(shifted, dataAsPairs); compare(segment, dataAsPairs); } @@ -176,9 +165,6 @@ class LongArraySortNTest { basic.mergeSortN(2, 0, size, Path.of("/tmp")); assertTrue(basic.isSortedN(2, 0, size)); - paged.mergeSortN(2, 0, size, Path.of("/tmp")); - assertTrue(paged.isSortedN(2, 0, size)); - shifted.mergeSortN(2, 0, size, Path.of("/tmp")); assertTrue(shifted.isSortedN(2, 0, size)); @@ -186,7 +172,6 @@ class LongArraySortNTest { assertTrue(segment.isSortedN(2, 0, size)); compare(basic, dataAsPairs); - compare(paged, dataAsPairs); compare(shifted, dataAsPairs); compare(segment, dataAsPairs); } diff --git a/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortTest.java index 5c5530b0..2cfde5a7 100644 --- a/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortTest.java +++ b/code/libraries/array/test/nu/marginalia/array/algo/LongArraySortTest.java @@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class LongArraySortTest { LongArray basic; - LongArray paged; LongArray shifted; LongArray segment; @@ -32,7 +31,6 @@ class LongArraySortTest { @BeforeEach public void setUp() { basic = LongArray.allocate(size); - paged = LongArray.allocate(size); shifted = LongArray.allocate(size+30).shifted(30); segment = LongArrayFactory.onHeapConfined(size + 30).shifted(30); @@ -46,12 +44,10 @@ class LongArraySortTest { } basic.set(0, values); - paged.set(0, values); shifted.set(0, values); segment.set(0, values); basic.transformEach(0, size, (i, old) -> values[(int) i]); - paged.transformEach(0, size, (i, old) -> values[(int) i]); shifted.transformEach(0, size, (i, old) -> values[(int) i]); segment.transformEach(0, size, (i, old) -> values[(int) i]); } @@ -68,7 +64,7 @@ class LongArraySortTest { array.set(1, 4); array.set(2, 3); array.set(3, 2); - array.quickSortNative(0, 4); + array.sort(0, 4); assertTrue(array.isSorted(0, 4)); assertEquals(1, array.get(0)); assertEquals(2, array.get(1)); @@ -76,13 +72,14 @@ class LongArraySortTest { assertEquals(4, array.get(3)); array.set(2, 5); - array.quickSortNative(2, 4); + array.sort(2, 4); assertEquals(4, array.get(2)); assertEquals(5, array.get(3)); assertTrue(array.isSorted(2, 4)); } + @Test public void quickSortStressTest() throws IOException { LongArray array = LongArray.allocate(65536); @@ -92,7 +89,7 @@ class LongArraySortTest { @Test public void nativeSortTest() throws IOException { LongArray array = LongArray.allocate(65536); - sortAlgorithmTester(array, LongArraySort::quickSortNative); + sortAlgorithmTester(array, LongArraySort::sort); } @@ -166,41 +163,33 @@ class LongArraySortTest { @Test void insertionSort() { - basic.insertionSort(0, size); + LongArraySort.insertionSort(basic, 0, size); assertTrue(basic.isSorted(0, 128)); - paged.insertionSort(0, size); - assertTrue(paged.isSorted(0, 128)); - - shifted.insertionSort(0, size); + LongArraySort.insertionSort(shifted, 0, size); assertTrue(shifted.isSorted(0, 128)); - segment.insertionSort(0, size); + LongArraySort.insertionSort(segment, 0, size); assertTrue(segment.isSorted(0, 128)); verifyValuesPresent(basic); - verifyValuesPresent(paged); verifyValuesPresent(shifted); verifyValuesPresent(segment); } @Test void quickSort() { - basic.quickSort(0, size); + basic.sort(0, size); assertTrue(basic.isSorted(0, size)); - paged.quickSort(0, size); - assertTrue(paged.isSorted(0, size)); - - shifted.quickSort(0, size); + shifted.sort(0, size); assertTrue(shifted.isSorted(0, size)); - segment.quickSort(0, size); + segment.sort(0, size); assertTrue(segment.isSorted(0, size)); verifyValuesPresent(basic); - verifyValuesPresent(paged); verifyValuesPresent(shifted); verifyValuesPresent(segment); @@ -211,9 +200,6 @@ class LongArraySortTest { basic.mergeSort(0, size, Path.of("/tmp")); assertTrue(basic.isSorted(0, size)); - paged.mergeSort(0, size, Path.of("/tmp")); - assertTrue(paged.isSorted(0, size)); - shifted.mergeSort(0, size, Path.of("/tmp")); assertTrue(shifted.isSorted(0, size)); @@ -221,7 +207,6 @@ class LongArraySortTest { assertTrue(segment.isSorted(0, size)); verifyValuesPresent(basic); - verifyValuesPresent(paged); verifyValuesPresent(shifted); verifyValuesPresent(segment); } @@ -231,7 +216,7 @@ class LongArraySortTest { var array = LongArray.allocate(1000); var random = new Random(); array.transformEach(0, 1000, (i, val) -> random.nextInt(0, 2000)); - array.quickSort(0, 1000); + array.sort(0, 1000); Set expectedValues = new HashSet<>(); array.forEach(0, 1000, (i, v) -> expectedValues.add(v)); diff --git a/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformations2Test.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformations2Test.java index 01af58cc..78fc879b 100644 --- a/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformations2Test.java +++ b/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformations2Test.java @@ -11,7 +11,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; class LongArrayTransformations2Test { LongArray basic; - LongArray paged; LongArray shifted; final int size = 1026; @@ -19,7 +18,6 @@ class LongArrayTransformations2Test { @BeforeEach public void setUp() { basic = LongArray.allocate(size); - paged = LongArray.allocate(size); shifted = LongArray.allocate(size+30).shifted(30); long[] vals = new long[size]; @@ -27,26 +25,22 @@ class LongArrayTransformations2Test { vals[i] = i+10; } basic.set(0, vals); - paged.set(0, vals); shifted.set(0, vals); } @Test void forEach() { foreachTester(basic); - foreachTester(paged); foreachTester(shifted); } @Test void transformEach() { transformTester(basic); - transformTester(paged); transformTester(shifted); } @Test void transformEachIO() throws IOException { transformTesterIO(basic); - transformTesterIO(paged); transformTesterIO(shifted); } diff --git a/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformationsTest.java b/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformationsTest.java index 92fb3842..ad1cb495 100644 --- a/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformationsTest.java +++ b/code/libraries/array/test/nu/marginalia/array/algo/LongArrayTransformationsTest.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; class LongArrayTransformationsTest { LongArray basic; - LongArray paged; LongArray shifted; final int size = 1026; @@ -18,12 +17,10 @@ class LongArrayTransformationsTest { @BeforeEach public void setUp() { basic = LongArray.allocate(size); - paged = LongArray.allocate(size); shifted = LongArray.allocate(size+30).shifted(30); for (int i = 0; i < basic.size(); i++) { basic.set(i, 3L*i); - paged.set(i, 3L*i); shifted.set(i, 3L*i); } } @@ -31,28 +28,24 @@ class LongArrayTransformationsTest { @Test void transformEach() { transformTester(basic); - transformTester(paged); transformTester(shifted); } @Test void transformEachIO() throws IOException { transformTesterIO(basic); - transformTesterIO(paged); transformTesterIO(shifted); } @Test void fold() { assertEquals(3*(5+6+7+8+9), basic.fold(0, 5, 10, Long::sum)); - assertEquals(3*(5+6+7+8+9), paged.fold(0, 5, 10, Long::sum)); assertEquals(3*(5+6+7+8+9), shifted.fold(0, 5, 10, Long::sum)); } @Test void foldIO() throws IOException { assertEquals(3*(5+6+7+8+9), basic.foldIO(0, 5, 10, Long::sum)); - assertEquals(3*(5+6+7+8+9), paged.foldIO(0, 5, 10, Long::sum)); assertEquals(3*(5+6+7+8+9), shifted.foldIO(0, 5, 10, Long::sum)); } diff --git a/code/libraries/array/test/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java b/code/libraries/array/test/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java deleted file mode 100644 index c0506df0..00000000 --- a/code/libraries/array/test/nu/marginalia/array/scheme/ArrayPartitioningSchemeTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.array.scheme; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class ArrayPartitioningSchemeTest { - - @Test - public void testPo2() { - var p2 = new PowerOf2PartitioningScheme(64); - var seq = new SequentialPartitioningScheme(64); - - for (int i = 0; i < 512; i++) { - Assertions.assertEquals(p2.getPage(i), seq.getPage(i), "Unexpected buffer @ " + i); - Assertions.assertEquals(p2.getOffset(i), seq.getOffset(i), "Unexpected offset @ " + i); - Assertions.assertEquals(p2.isSamePage(i, i+1), seq.isSamePage(i, i+1), "Unexpected value @ " + i); - } - } -} \ No newline at end of file diff --git a/code/libraries/btree/java/nu/marginalia/btree/BTreeReader.java b/code/libraries/btree/java/nu/marginalia/btree/BTreeReader.java index 3b17bd26..064f12a1 100644 --- a/code/libraries/btree/java/nu/marginalia/btree/BTreeReader.java +++ b/code/libraries/btree/java/nu/marginalia/btree/BTreeReader.java @@ -1,7 +1,7 @@ package nu.marginalia.btree; import nu.marginalia.array.LongArray; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.btree.model.BTreeContext; import nu.marginalia.btree.model.BTreeHeader; diff --git a/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java index be24de10..d847e3ad 100644 --- a/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java +++ b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java @@ -1,7 +1,7 @@ package nu.marginalia.btree; import nu.marginalia.array.LongArray; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.btree.model.BTreeBlockSize; import nu.marginalia.btree.model.BTreeContext; import nu.marginalia.util.NextPrimeUtil; @@ -9,7 +9,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.Arrays; import static org.junit.jupiter.api.Assertions.assertArrayEquals; diff --git a/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java index fc3b71df..db8d9460 100644 --- a/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java +++ b/code/libraries/btree/test/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java @@ -1,7 +1,7 @@ package nu.marginalia.btree; import nu.marginalia.array.LongArray; -import nu.marginalia.array.buffer.LongQueryBuffer; +import nu.marginalia.array.page.LongQueryBuffer; import nu.marginalia.btree.model.BTreeBlockSize; import nu.marginalia.btree.model.BTreeContext; import nu.marginalia.util.NextPrimeUtil; @@ -9,7 +9,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.Arrays; import static org.junit.jupiter.api.Assertions.assertArrayEquals;