From a8cc98a0f62e2c3a1ea7aea523d666afbe59b59f Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 29 Sep 2024 17:20:29 +0200 Subject: [PATCH] (index) Fix write offset calculation in PrioDocIdsTransformer Adjust the write offset calculation by adding the position of the write buffer. Updated the test to validate the transformation process and ensure correctness of output file positions. --- .../prio/PrioDocIdsTransformer.java | 2 +- .../prio/PrioDocIdsTransformerTest.java | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioDocIdsTransformer.java b/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioDocIdsTransformer.java index 3072ffb8..92c3b8aa 100644 --- a/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioDocIdsTransformer.java +++ b/code/index/index-reverse/java/nu/marginalia/index/construction/prio/PrioDocIdsTransformer.java @@ -35,7 +35,7 @@ public class PrioDocIdsTransformer implements LongArrayTransformations.LongIOTra public long transform(long pos, long endL) throws IOException { final int sizeL = (int) ((endL - startL)); - final long startOffsetB = writeOffsetB; + final long startOffsetB = writeOffsetB + writeBuffer.position(); if (sizeL == 0) throw new IllegalStateException("Empty range"); diff --git a/code/index/index-reverse/test/nu/marginalia/index/construction/prio/PrioDocIdsTransformerTest.java b/code/index/index-reverse/test/nu/marginalia/index/construction/prio/PrioDocIdsTransformerTest.java index e4ced16d..494a6ff6 100644 --- a/code/index/index-reverse/test/nu/marginalia/index/construction/prio/PrioDocIdsTransformerTest.java +++ b/code/index/index-reverse/test/nu/marginalia/index/construction/prio/PrioDocIdsTransformerTest.java @@ -6,8 +6,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -16,7 +14,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; class PrioDocIdsTransformerTest { @@ -52,6 +51,15 @@ class PrioDocIdsTransformerTest { buffer.putLong(UrlIdCodec.encodeId(4, 51) | 0x7000_0000_0000_0000L); writeChannel.write(buffer.flip()); + + buffer.clear(); + + buffer.putLong(UrlIdCodec.encodeId(0, 0)); + buffer.putLong(UrlIdCodec.encodeId(0, 1)); + buffer.putLong(UrlIdCodec.encodeId(1, 0)); + buffer.putLong(UrlIdCodec.encodeId(4, 51) | 0x7000_0000_0000_0000L); + + writeChannel.write(buffer.flip()); } try (var writeChannel = (FileChannel) Files.newByteChannel(outputFile, StandardOpenOption.WRITE); @@ -60,7 +68,12 @@ class PrioDocIdsTransformerTest { { // Transform two segments of the input file and write them to the output file with prefixed sizes - transformer.transform(0, 4); + long pos1 = transformer.transform(0, 4); + long pos2 = transformer.transform(1, 8); + + // The functions return the positions in the output file, which should be non-zero for all but the first segment + assertEquals(0, pos1); + assertNotEquals(0, pos2); } byte[] bytes = Files.readAllBytes(outputFile);