Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(long-array) Return slices SegmentLongArray of itself for range() &c

Browse Source
This commit is contained in:
Viktor Lofgren 2023-09-24 11:31:54 +02:00
parent a144749a8d
commit cd12f49fc0
6 changed files with 38 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java
View File

@ -107,8 +107,8 @@ class ReversePreindexFinalizeTest {
System.out.println(Files.size(wordsFile)); System.out.println(Files.size(wordsFile));
System.out.println(Files.size(docsFile)); System.out.println(Files.size(docsFile));
var docsArray = LongArray.mmapRead(docsFile); var docsArray = LongArrayFactory.mmapForReadingConfined(docsFile);
var wordsArray = LongArray.mmapRead(wordsFile); var wordsArray = LongArrayFactory.mmapForReadingConfined(wordsFile);
var wordsHeader = BTreeReader.readHeader(wordsArray, 0); var wordsHeader = BTreeReader.readHeader(wordsArray, 0);

4
code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java
View File

@ -59,10 +59,10 @@ public interface LongArray extends LongArrayBase, LongArrayTransformations, Long
return PagingLongArray.mapFileReadWrite(DEFAULT_PARTITIONING_SCHEME, path, size); return PagingLongArray.mapFileReadWrite(DEFAULT_PARTITIONING_SCHEME, path, size);
} }
default ShiftedLongArray shifted(long offset) { default LongArray shifted(long offset) {
return new ShiftedLongArray(offset, this); return new ShiftedLongArray(offset, this);
} }
default ShiftedLongArray range(long start, long end) { default LongArray range(long start, long end) {
return new ShiftedLongArray(start, end, this); return new ShiftedLongArray(start, end, this);
} }

2
code/libraries/array/src/main/java/nu/marginalia/array/page/PagingLongArray.java
View File

@ -547,7 +547,7 @@ public class PagingLongArray extends AbstractPagingArray<LongArrayPage, LongBuff
return pages[partitioningScheme.getPage(forOffset)]; return pages[partitioningScheme.getPage(forOffset)];
} }
public ShiftedLongArray range(long start, long end) { public LongArray range(long start, long end) {
if (partitioningScheme.isSamePage(start, end)) { if (partitioningScheme.isSamePage(start, end)) {
return pages[partitioningScheme.getPage(start)] return pages[partitioningScheme.getPage(start)]
.range(partitioningScheme.getOffset(start), .range(partitioningScheme.getOffset(start),

37
code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentLongArray.java
View File

@ -8,7 +8,6 @@ import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.lang.foreign.Arena; import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment; import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.LongBuffer; import java.nio.LongBuffer;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
@ -17,6 +16,8 @@ import java.nio.file.OpenOption;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import static java.lang.foreign.ValueLayout.JAVA_LONG;
public class SegmentLongArray implements PartitionPage, LongArray { public class SegmentLongArray implements PartitionPage, LongArray {
@Nullable @Nullable
@ -58,8 +59,8 @@ public class SegmentLongArray implements PartitionPage, LongArray {
try (var channel = (FileChannel) Files.newByteChannel(file, openOptions)) { try (var channel = (FileChannel) Files.newByteChannel(file, openOptions)) {
return channel.map(mode, return channel.map(mode,
WORD_SIZE * offset, JAVA_LONG.byteSize() * offset,
WORD_SIZE * size, JAVA_LONG.byteSize() * size,
arena); arena);
} }
catch (IOException ex) { catch (IOException ex) {
@ -67,10 +68,26 @@ public class SegmentLongArray implements PartitionPage, LongArray {
} }
} }
@Override
public LongArray range(long start, long end) {
return new SegmentLongArray(
segment.asSlice(
start * JAVA_LONG.byteSize(),
(end-start) * JAVA_LONG.byteSize()),
null);
}
@Override
public LongArray shifted(long start) {
return new SegmentLongArray(
segment.asSlice(start * JAVA_LONG.byteSize()),
null);
}
@Override @Override
public long get(long at) { public long get(long at) {
try { try {
return segment.getAtIndex(ValueLayout.JAVA_LONG, at); return segment.getAtIndex(JAVA_LONG, at);
} }
catch (IndexOutOfBoundsException ex) { catch (IndexOutOfBoundsException ex) {
throw new IndexOutOfBoundsException("@" + at + "(" + 0 + ":" + segment.byteSize()/8 + ")"); throw new IndexOutOfBoundsException("@" + at + "(" + 0 + ":" + segment.byteSize()/8 + ")");
@ -80,13 +97,13 @@ public class SegmentLongArray implements PartitionPage, LongArray {
@Override @Override
public void get(long start, long end, long[] buffer) { public void get(long start, long end, long[] buffer) {
for (int i = 0; i < end - start; i++) { for (int i = 0; i < end - start; i++) {
buffer[i] = segment.getAtIndex(ValueLayout.JAVA_LONG, start + i); buffer[i] = segment.getAtIndex(JAVA_LONG, start + i);
} }
} }
@Override @Override
public void set(long at, long val) { public void set(long at, long val) {
segment.setAtIndex(ValueLayout.JAVA_LONG, at, val); segment.setAtIndex(JAVA_LONG, at, val);
} }
@Override @Override
@ -106,7 +123,7 @@ public class SegmentLongArray implements PartitionPage, LongArray {
@Override @Override
public long size() { public long size() {
return segment.byteSize() / 8; return segment.byteSize() / JAVA_LONG.byteSize();
} }
@Override @Override
@ -135,12 +152,12 @@ public class SegmentLongArray implements PartitionPage, LongArray {
@Override @Override
public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException { public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException {
long index = arrayStart * WORD_SIZE; long index = arrayStart * JAVA_LONG.byteSize();
long length = (arrayEnd - arrayStart) * WORD_SIZE; long length = (arrayEnd - arrayStart) * JAVA_LONG.byteSize();
var bufferSlice = segment.asSlice(index, length).asByteBuffer(); var bufferSlice = segment.asSlice(index, length).asByteBuffer();
long startPos = sourceStart * WORD_SIZE; long startPos = sourceStart * JAVA_LONG.byteSize();
while (bufferSlice.position() < bufferSlice.capacity()) { while (bufferSlice.position() < bufferSlice.capacity()) {
source.read(bufferSlice, startPos + bufferSlice.position()); source.read(bufferSlice, startPos + bufferSlice.position());
} }

8
code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java
View File

@ -12,7 +12,7 @@ import static java.lang.Math.min;
public class BTreeReader { public class BTreeReader {
private final LongArray index; private final LongArray index;
private final ShiftedLongArray data; private final LongArray data;
public final BTreeContext ctx; public final BTreeContext ctx;
private final BTreeHeader header; private final BTreeHeader header;
@ -148,7 +148,7 @@ public class BTreeReader {
long searchStart = 0; long searchStart = 0;
for (int i = 0; i < keys.length; i++) { for (int i = 0; i < keys.length; i++) {
long key = keys[i]; long key = keys[i];
searchStart = data.binarySearchN(ctx.entrySize, key, searchStart, data.size); searchStart = data.binarySearchN(ctx.entrySize, key, searchStart, data.size());
if (searchStart < 0) { if (searchStart < 0) {
searchStart = LongArraySearch.decodeSearchMiss(searchStart); searchStart = LongArraySearch.decodeSearchMiss(searchStart);
} }
@ -273,7 +273,7 @@ public class BTreeReader {
long searchEnd = dataOffset + (int) min(remainingTotal, remainingBlock); long searchEnd = dataOffset + (int) min(remainingTotal, remainingBlock);
data.range(dataOffset, searchEnd).retainN(buffer, ctx.entrySize, boundary); data.retainN(buffer, ctx.entrySize, boundary, dataOffset, searchEnd);
} }
} }
else { else {
@ -297,7 +297,7 @@ public class BTreeReader {
long searchEnd = dataOffset + (int) min(remainingTotal, remainingBlock); long searchEnd = dataOffset + (int) min(remainingTotal, remainingBlock);
data.range(dataOffset, searchEnd).rejectN(buffer, ctx.entrySize, boundary); data.rejectN(buffer, ctx.entrySize, boundary, dataOffset, searchEnd);
} }
} }
else { else {

4
code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java
View File

@ -41,7 +41,7 @@ public class BTreeWriter {
throw new IllegalStateException("Dog ear was not overwritten: " + header); throw new IllegalStateException("Dog ear was not overwritten: " + header);
} }
assert slice.isSortedN(ctx.entrySize) : "Provided data was not sorted"; assert slice.isSortedN(ctx.entrySize, 0, (long) numEntries * ctx.entrySize) : "Provided data was not sorted";
if (header.layers() >= 1) { // Omit layer if data fits within a single block if (header.layers() >= 1) { // Omit layer if data fits within a single block
writeIndex(header); writeIndex(header);
@ -56,7 +56,7 @@ public class BTreeWriter {
map.set(offset+2, header.dataOffsetLongs()); map.set(offset+2, header.dataOffsetLongs());
} }
private BTreeDogEar createDogEar(BTreeContext ctx, BTreeHeader header, ShiftedLongArray slice) { private BTreeDogEar createDogEar(BTreeContext ctx, BTreeHeader header, LongArray slice) {
if (BTreeWriter.class.desiredAssertionStatus()) { if (BTreeWriter.class.desiredAssertionStatus()) {
return BTreeDogEar.create(ctx, header, slice); return BTreeDogEar.create(ctx, header, slice);
} }