Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(long-array) Return slices SegmentLongArray of itself for range() &c

Browse Source
This commit is contained in:
Viktor Lofgren 2023-09-24 11:31:54 +02:00
parent a144749a8d
commit cd12f49fc0
6 changed files with 38 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
code/features-index/index-reverse/src/test/java/nu/marginalia/index/construction/ReversePreindexFinalizeTest.java
View File

@ -107,8 +107,8 @@ class ReversePreindexFinalizeTest {
System.out.println(Files.size(wordsFile));
System.out.println(Files.size(docsFile));
var docsArray = LongArray.mmapRead(docsFile);
var wordsArray = LongArray.mmapRead(wordsFile);
var docsArray = LongArrayFactory.mmapForReadingConfined(docsFile);
var wordsArray = LongArrayFactory.mmapForReadingConfined(wordsFile);
var wordsHeader = BTreeReader.readHeader(wordsArray, 0);

4
code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java
View File

@ -59,10 +59,10 @@ public interface LongArray extends LongArrayBase, LongArrayTransformations, Long
return PagingLongArray.mapFileReadWrite(DEFAULT_PARTITIONING_SCHEME, path, size);
}
default ShiftedLongArray shifted(long offset) {
default LongArray shifted(long offset) {
return new ShiftedLongArray(offset, this);
}
default ShiftedLongArray range(long start, long end) {
default LongArray range(long start, long end) {
return new ShiftedLongArray(start, end, this);
}

2
code/libraries/array/src/main/java/nu/marginalia/array/page/PagingLongArray.java
View File

@ -547,7 +547,7 @@ public class PagingLongArray extends AbstractPagingArray<LongArrayPage, LongBuff
return pages[partitioningScheme.getPage(forOffset)];
}
public ShiftedLongArray range(long start, long end) {
public LongArray range(long start, long end) {
if (partitioningScheme.isSamePage(start, end)) {
return pages[partitioningScheme.getPage(start)]
.range(partitioningScheme.getOffset(start),

37
code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentLongArray.java
View File

@ -8,7 +8,6 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteBuffer;
import java.nio.LongBuffer;
import java.nio.channels.FileChannel;
@ -17,6 +16,8 @@ import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import static java.lang.foreign.ValueLayout.JAVA_LONG;
public class SegmentLongArray implements PartitionPage, LongArray {
@Nullable
@ -58,8 +59,8 @@ public class SegmentLongArray implements PartitionPage, LongArray {
try (var channel = (FileChannel) Files.newByteChannel(file, openOptions)) {
return channel.map(mode,
WORD_SIZE * offset,
WORD_SIZE * size,
JAVA_LONG.byteSize() * offset,
JAVA_LONG.byteSize() * size,
arena);
}
catch (IOException ex) {
@ -67,10 +68,26 @@ public class SegmentLongArray implements PartitionPage, LongArray {
}
}
@Override
public LongArray range(long start, long end) {
return new SegmentLongArray(
segment.asSlice(
start * JAVA_LONG.byteSize(),
(end-start) * JAVA_LONG.byteSize()),
null);
}
@Override
public LongArray shifted(long start) {
return new SegmentLongArray(
segment.asSlice(start * JAVA_LONG.byteSize()),
null);
}
@Override
public long get(long at) {
try {
return segment.getAtIndex(ValueLayout.JAVA_LONG, at);
return segment.getAtIndex(JAVA_LONG, at);
}
catch (IndexOutOfBoundsException ex) {
throw new IndexOutOfBoundsException("@" + at + "(" + 0 + ":" + segment.byteSize()/8 + ")");
@ -80,13 +97,13 @@ public class SegmentLongArray implements PartitionPage, LongArray {
@Override
public void get(long start, long end, long[] buffer) {
for (int i = 0; i < end - start; i++) {
buffer[i] = segment.getAtIndex(ValueLayout.JAVA_LONG, start + i);
buffer[i] = segment.getAtIndex(JAVA_LONG, start + i);
}
}
@Override
public void set(long at, long val) {
segment.setAtIndex(ValueLayout.JAVA_LONG, at, val);
segment.setAtIndex(JAVA_LONG, at, val);
}
@Override
@ -106,7 +123,7 @@ public class SegmentLongArray implements PartitionPage, LongArray {
@Override
public long size() {
return segment.byteSize() / 8;
return segment.byteSize() / JAVA_LONG.byteSize();
}
@Override
@ -135,12 +152,12 @@ public class SegmentLongArray implements PartitionPage, LongArray {
@Override
public void transferFrom(FileChannel source, long sourceStart, long arrayStart, long arrayEnd) throws IOException {
long index = arrayStart * WORD_SIZE;
long length = (arrayEnd - arrayStart) * WORD_SIZE;
long index = arrayStart * JAVA_LONG.byteSize();
long length = (arrayEnd - arrayStart) * JAVA_LONG.byteSize();
var bufferSlice = segment.asSlice(index, length).asByteBuffer();
long startPos = sourceStart * WORD_SIZE;
long startPos = sourceStart * JAVA_LONG.byteSize();
while (bufferSlice.position() < bufferSlice.capacity()) {
source.read(bufferSlice, startPos + bufferSlice.position());
}

8
code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java
View File

@ -12,7 +12,7 @@ import static java.lang.Math.min;
public class BTreeReader {
private final LongArray index;
private final ShiftedLongArray data;
private final LongArray data;
public final BTreeContext ctx;
private final BTreeHeader header;
@ -148,7 +148,7 @@ public class BTreeReader {
long searchStart = 0;
for (int i = 0; i < keys.length; i++) {
long key = keys[i];
searchStart = data.binarySearchN(ctx.entrySize, key, searchStart, data.size);
searchStart = data.binarySearchN(ctx.entrySize, key, searchStart, data.size());
if (searchStart < 0) {
searchStart = LongArraySearch.decodeSearchMiss(searchStart);
}
@ -273,7 +273,7 @@ public class BTreeReader {
long searchEnd = dataOffset + (int) min(remainingTotal, remainingBlock);
data.range(dataOffset, searchEnd).retainN(buffer, ctx.entrySize, boundary);
data.retainN(buffer, ctx.entrySize, boundary, dataOffset, searchEnd);
}
}
else {
@ -297,7 +297,7 @@ public class BTreeReader {
long searchEnd = dataOffset + (int) min(remainingTotal, remainingBlock);
data.range(dataOffset, searchEnd).rejectN(buffer, ctx.entrySize, boundary);
data.rejectN(buffer, ctx.entrySize, boundary, dataOffset, searchEnd);
}
}
else {

4
code/libraries/btree/src/main/java/nu/marginalia/btree/BTreeWriter.java
View File

@ -41,7 +41,7 @@ public class BTreeWriter {
throw new IllegalStateException("Dog ear was not overwritten: " + header);
}
assert slice.isSortedN(ctx.entrySize) : "Provided data was not sorted";
assert slice.isSortedN(ctx.entrySize, 0, (long) numEntries * ctx.entrySize) : "Provided data was not sorted";
if (header.layers() >= 1) { // Omit layer if data fits within a single block
writeIndex(header);
@ -56,7 +56,7 @@ public class BTreeWriter {
map.set(offset+2, header.dataOffsetLongs());
}
private BTreeDogEar createDogEar(BTreeContext ctx, BTreeHeader header, ShiftedLongArray slice) {
private BTreeDogEar createDogEar(BTreeContext ctx, BTreeHeader header, LongArray slice) {
if (BTreeWriter.class.desiredAssertionStatus()) {
return BTreeDogEar.create(ctx, header, slice);
}