Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(slop) Add columnDesc information to the column readers and writers, and correct a few broken position() implementations

Browse Source 
Added a test that should find any additional broken implementations, as it's very important that this function is correct.
This commit is contained in:
Viktor Lofgren 2024-07-27 14:35:09 +02:00
parent 2e1f669aea
commit f8684118f3
18 changed files with 576 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
code/libraries/coded-sequence/java/nu/marginalia/sequence/slop/GammaCodedSequenceColumn.java
View File

@ -22,20 +22,20 @@ public class GammaCodedSequenceColumn {
public static ColumnType<GammaCodedSequenceReader, GammaCodedSequenceWriter> TYPE = ColumnType.register("s8[]+gcs", ByteOrder.nativeOrder(), GammaCodedSequenceColumn::open, GammaCodedSequenceColumn::create);
public static GammaCodedSequenceReader open(Path path, ColumnDesc name) throws IOException {
return new Reader(
Storage.reader(path, name, false), // note we must never pass aligned=true here, as the data is not guaranteed alignment
VarintColumn.open(path, name.createSupplementaryColumn(ColumnFunction.DATA_LEN,
public static GammaCodedSequenceReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(columnDesc,
Storage.reader(path, columnDesc, false), // note we must never pass aligned=true here, as the data is not guaranteed alignment
VarintColumn.open(path, columnDesc.createSupplementaryColumn(ColumnFunction.DATA_LEN,
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
);
}
public static GammaCodedSequenceWriter create(Path path, ColumnDesc name) throws IOException {
return new Writer(
Storage.writer(path, name),
VarintColumn.create(path, name.createSupplementaryColumn(ColumnFunction.DATA_LEN,
public static GammaCodedSequenceWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(columnDesc,
Storage.writer(path, columnDesc),
VarintColumn.create(path, columnDesc.createSupplementaryColumn(ColumnFunction.DATA_LEN,
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
@ -44,16 +44,23 @@ public class GammaCodedSequenceColumn {
private static class Writer implements GammaCodedSequenceWriter {
private final VarintColumnWriter indexWriter;
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
public Writer(StorageWriter storage,
public Writer(ColumnDesc<?, ?> columnDesc,
StorageWriter storage,
VarintColumnWriter indexWriter)
{
this.columnDesc = columnDesc;
this.storage = storage;
this.indexWriter = indexWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
@Override
public void put(GammaCodedSequence sequence) throws IOException {
@ -76,13 +83,20 @@ public class GammaCodedSequenceColumn {
private static class Reader implements GammaCodedSequenceReader {
private final VarintColumnReader indexReader;
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader reader, VarintColumnReader indexReader) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader reader, VarintColumnReader indexReader) throws IOException {
this.columnDesc = columnDesc;
this.storage = reader;
this.indexReader = indexReader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
@Override
public void skip(long positions) throws IOException {
for (int i = 0; i < positions; i++) {

5
code/libraries/slop/java/nu/marginalia/slop/column/ColumnReader.java
View File

@ -1,8 +1,13 @@
package nu.marginalia.slop.column;
import nu.marginalia.slop.desc.ColumnDesc;
import java.io.IOException;
public interface ColumnReader {
ColumnDesc<?, ?> columnDesc();
long position() throws IOException;
void skip(long positions) throws IOException;

4
code/libraries/slop/java/nu/marginalia/slop/column/ColumnWriter.java
View File

@ -1,8 +1,12 @@
package nu.marginalia.slop.column;
import nu.marginalia.slop.desc.ColumnDesc;
import java.io.IOException;
public interface ColumnWriter {
ColumnDesc<?, ?> columnDesc();
/** Return the current record index in the column */
long position();

32
code/libraries/slop/java/nu/marginalia/slop/column/array/ByteArrayColumn.java
View File

@ -15,22 +15,24 @@ import java.nio.file.Path;
public class ByteArrayColumn {
public static ByteArrayColumnReader open(Path path, ColumnDesc name) throws IOException {
public static ByteArrayColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(
Storage.reader(path, name, true),
columnDesc,
Storage.reader(path, columnDesc, true),
VarintColumn.open(path,
name.createSupplementaryColumn(name.function().lengthsTable(),
columnDesc.createSupplementaryColumn(columnDesc.function().lengthsTable(),
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
);
}
public static ByteArrayColumnWriter create(Path path, ColumnDesc name) throws IOException {
public static ByteArrayColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(
Storage.writer(path, name),
columnDesc,
Storage.writer(path, columnDesc),
VarintColumn.create(path,
name.createSupplementaryColumn(name.function().lengthsTable(),
columnDesc.createSupplementaryColumn(columnDesc.function().lengthsTable(),
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
@ -38,16 +40,23 @@ public class ByteArrayColumn {
}
private static class Writer implements ByteArrayColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private final VarintColumnWriter lengthsWriter;
private long position = 0;
public Writer(StorageWriter storage, VarintColumnWriter lengthsWriter) throws IOException {
public Writer(ColumnDesc<?, ?> columnDesc, StorageWriter storage, VarintColumnWriter lengthsWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
this.lengthsWriter = lengthsWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(byte[] value) throws IOException {
position ++;
storage.putBytes(value);
@ -65,14 +74,21 @@ public class ByteArrayColumn {
}
private static class Reader implements ByteArrayColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
private final VarintColumnReader lengthsReader;
public Reader(StorageReader storage, VarintColumnReader lengthsReader) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage, VarintColumnReader lengthsReader) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
this.lengthsReader = lengthsReader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public byte[] get() throws IOException {
int length = (int) lengthsReader.get();
byte[] ret = new byte[length];

32
code/libraries/slop/java/nu/marginalia/slop/column/array/IntArrayColumn.java
View File

@ -15,18 +15,20 @@ import java.nio.file.Path;
public class IntArrayColumn {
public static IntArrayColumnReader open(Path path, ColumnDesc name) throws IOException {
return new Reader(Storage.reader(path, name, true),
VarintColumn.open(path, name.createSupplementaryColumn(name.function().lengthsTable(),
public static IntArrayColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(columnDesc,
Storage.reader(path, columnDesc, true),
VarintColumn.open(path, columnDesc.createSupplementaryColumn(columnDesc.function().lengthsTable(),
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
);
}
public static IntArrayColumnWriter create(Path path, ColumnDesc name) throws IOException {
return new Writer(Storage.writer(path, name),
VarintColumn.create(path, name.createSupplementaryColumn(name.function().lengthsTable(),
public static IntArrayColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(columnDesc,
Storage.writer(path, columnDesc),
VarintColumn.create(path, columnDesc.createSupplementaryColumn(columnDesc.function().lengthsTable(),
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
@ -34,14 +36,21 @@ public class IntArrayColumn {
}
private static class Writer implements IntArrayColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private final VarintColumnWriter lengthsWriter;
public Writer(StorageWriter storage, VarintColumnWriter lengthsWriter) throws IOException {
public Writer(ColumnDesc<?, ?> columnDesc, StorageWriter storage, VarintColumnWriter lengthsWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
this.lengthsWriter = lengthsWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(int[] value) throws IOException {
storage.putInts(value);
lengthsWriter.put(value.length);
@ -58,14 +67,21 @@ public class IntArrayColumn {
}
private static class Reader implements IntArrayColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
private final VarintColumnReader lengthsReader;
public Reader(StorageReader storage, VarintColumnReader lengthsReader) {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage, VarintColumnReader lengthsReader) {
this.columnDesc = columnDesc;
this.storage = storage;
this.lengthsReader = lengthsReader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public int[] get() throws IOException {
int length = (int) lengthsReader.get();
int[] ret = new int[length];

34
code/libraries/slop/java/nu/marginalia/slop/column/array/LongArrayColumn.java
View File

@ -15,18 +15,22 @@ import java.nio.file.Path;
public class LongArrayColumn {
public static LongArrayColumnReader open(Path path, ColumnDesc name) throws IOException {
return new LongArrayColumn.Reader(Storage.reader(path, name, true),
VarintColumn.open(path, name.createSupplementaryColumn(name.function().lengthsTable(),
public static LongArrayColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new LongArrayColumn.Reader(
columnDesc,
Storage.reader(path, columnDesc, true),
VarintColumn.open(path, columnDesc.createSupplementaryColumn(columnDesc.function().lengthsTable(),
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
);
}
public static LongArrayColumnWriter create(Path path, ColumnDesc name) throws IOException {
return new LongArrayColumn.Writer(Storage.writer(path, name),
VarintColumn.create(path, name.createSupplementaryColumn(name.function().lengthsTable(),
public static LongArrayColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new LongArrayColumn.Writer(
columnDesc,
Storage.writer(path, columnDesc),
VarintColumn.create(path, columnDesc.createSupplementaryColumn(columnDesc.function().lengthsTable(),
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
@ -34,14 +38,21 @@ public class LongArrayColumn {
}
private static class Writer implements LongArrayColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private final VarintColumnWriter lengthsWriter;
public Writer(StorageWriter storage, VarintColumnWriter lengthsWriter) throws IOException {
public Writer(ColumnDesc<?,?> columnDesc, StorageWriter storage, VarintColumnWriter lengthsWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
this.lengthsWriter = lengthsWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(long[] value) throws IOException {
storage.putLongs(value);
lengthsWriter.put(value.length);
@ -58,14 +69,21 @@ public class LongArrayColumn {
}
private static class Reader implements LongArrayColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
private final VarintColumnReader lengthsReader;
public Reader(StorageReader storage, VarintColumnReader lengthsReader) {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage, VarintColumnReader lengthsReader) {
this.columnDesc = columnDesc;
this.storage = storage;
this.lengthsReader = lengthsReader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public long[] get() throws IOException {
int length = (int) lengthsReader.get();
long[] ret = new long[length];

35
code/libraries/slop/java/nu/marginalia/slop/column/dynamic/CustomBinaryColumn.java
View File

@ -13,20 +13,22 @@ import java.nio.file.Path;
public class CustomBinaryColumn {
public static CustomBinaryColumnReader open(Path path, ColumnDesc name) throws IOException {
public static CustomBinaryColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(
Storage.reader(path, name, false), // note we must never pass aligned=true here, as the data is not guaranteed alignment
VarintColumn.open(path, name.createSupplementaryColumn(ColumnFunction.DATA_LEN,
columnDesc,
Storage.reader(path, columnDesc, false), // note we must never pass aligned=true here, as the data is not guaranteed alignment
VarintColumn.open(path, columnDesc.createSupplementaryColumn(ColumnFunction.DATA_LEN,
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
);
}
public static CustomBinaryColumnWriter create(Path path, ColumnDesc name) throws IOException {
public static CustomBinaryColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(
Storage.writer(path, name),
VarintColumn.create(path, name.createSupplementaryColumn(ColumnFunction.DATA_LEN,
columnDesc,
Storage.writer(path, columnDesc),
VarintColumn.create(path, columnDesc.createSupplementaryColumn(ColumnFunction.DATA_LEN,
ColumnType.VARINT_LE,
StorageType.PLAIN)
)
@ -35,16 +37,24 @@ public class CustomBinaryColumn {
private static class Writer implements CustomBinaryColumnWriter {
private final VarintColumnWriter indexWriter;
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
public Writer(StorageWriter storage,
public Writer(ColumnDesc<?, ?> columnDesc,
StorageWriter storage,
VarintColumnWriter indexWriter)
{
this.columnDesc = columnDesc;
this.storage = storage;
this.indexWriter = indexWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
@Override
public RecordWriter next() throws IOException {
return new RecordWriter() {
@ -74,13 +84,20 @@ public class CustomBinaryColumn {
private static class Reader implements CustomBinaryColumnReader {
private final VarintColumnReader indexReader;
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader reader, VarintColumnReader indexReader) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader reader, VarintColumnReader indexReader) throws IOException {
this.columnDesc = columnDesc;
this.storage = reader;
this.indexReader = indexReader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
@Override
public void skip(long positions) throws IOException {
for (int i = 0; i < positions; i++) {

22
code/libraries/slop/java/nu/marginalia/slop/column/dynamic/VarintColumn.java
View File

@ -11,22 +11,29 @@ import java.nio.file.Path;
public class VarintColumn {
public static VarintColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(Storage.reader(path, columnDesc, true));
return new Reader(columnDesc, Storage.reader(path, columnDesc, true));
}
public static VarintColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(Storage.writer(path, columnDesc));
return new Writer(columnDesc, Storage.writer(path, columnDesc));
}
private static class Writer implements VarintColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter writer;
private long position = 0;
public Writer(StorageWriter writer) throws IOException {
public Writer(ColumnDesc<?,?> columnDesc, StorageWriter writer) throws IOException {
this.columnDesc = columnDesc;
this.writer = writer;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(long value) throws IOException {
position++;
@ -53,14 +60,21 @@ public class VarintColumn {
}
private static class Reader implements VarintColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader reader;
private long position = 0;
public Reader(StorageReader reader) throws IOException {
public Reader(ColumnDesc<?,?> columnDesc, StorageReader reader) throws IOException {
this.columnDesc = columnDesc;
this.reader = reader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public long get() throws IOException {
long value = 0;
int shift = 0;

22
code/libraries/slop/java/nu/marginalia/slop/column/primitive/ByteColumn.java
View File

@ -11,21 +11,28 @@ import java.nio.file.Path;
public class ByteColumn {
public static ByteColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(Storage.reader(path, columnDesc, true));
return new Reader(columnDesc, Storage.reader(path, columnDesc, true));
}
public static ByteColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(Storage.writer(path, columnDesc));
return new Writer(columnDesc, Storage.writer(path, columnDesc));
}
private static class Writer implements ByteColumnWriter {
private final ColumnDesc columnDesc;
private final StorageWriter storage;
private long position = 0;
public Writer(StorageWriter storageWriter) throws IOException {
public Writer(ColumnDesc columnDesc, StorageWriter storageWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(byte value) throws IOException {
storage.putByte(value);
position++;
@ -41,9 +48,11 @@ public class ByteColumn {
}
private static class Reader implements ByteColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader storage) throws IOException {
public Reader(ColumnDesc<?,?> columnDesc, StorageReader storage) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
}
@ -51,6 +60,11 @@ public class ByteColumn {
return storage.getByte();
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
@Override
public long position() throws IOException {
return storage.position();

26
code/libraries/slop/java/nu/marginalia/slop/column/primitive/CharColumn.java
View File

@ -11,28 +11,35 @@ import java.nio.file.Path;
public class CharColumn {
public static CharColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(Storage.reader(path, columnDesc, true));
return new Reader(columnDesc, Storage.reader(path, columnDesc, true));
}
public static CharColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(Storage.writer(path, columnDesc));
return new Writer(columnDesc, Storage.writer(path, columnDesc));
}
private static class Writer implements CharColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private long position = 0;
public Writer(StorageWriter storageWriter) throws IOException {
public Writer(ColumnDesc<?, ?> columnDesc, StorageWriter storageWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(char value) throws IOException {
storage.putChar(value);
position++;
}
public long position() {
return position / Character.BYTES;
return position;
}
public void close() throws IOException {
@ -41,9 +48,11 @@ public class CharColumn {
}
private static class Reader implements CharColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader storage) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
}
@ -51,9 +60,14 @@ public class CharColumn {
return storage.getChar();
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
@Override
public long position() throws IOException {
return storage.position();
return storage.position() / Character.BYTES;
}
@Override

26
code/libraries/slop/java/nu/marginalia/slop/column/primitive/DoubleColumn.java
View File

@ -11,28 +11,35 @@ import java.nio.file.Path;
public class DoubleColumn {
public static DoubleColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(Storage.reader(path, columnDesc, true));
return new Reader(columnDesc, Storage.reader(path, columnDesc, true));
}
public static DoubleColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(Storage.writer(path, columnDesc));
return new Writer(columnDesc, Storage.writer(path, columnDesc));
}
private static class Writer implements DoubleColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private long position = 0;
public Writer(StorageWriter storageWriter) throws IOException {
public Writer(ColumnDesc<?, ?> columnDesc, StorageWriter storageWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(double value) throws IOException {
storage.putDouble(value);
position++;
}
public long position() {
return position / Double.BYTES;
return position;
}
public void close() throws IOException {
@ -41,19 +48,26 @@ public class DoubleColumn {
}
private static class Reader implements DoubleColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader storage) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public double get() throws IOException {
return storage.getDouble();
}
@Override
public long position() throws IOException {
return storage.position();
return storage.position() / Double.BYTES;
}
@Override

22
code/libraries/slop/java/nu/marginalia/slop/column/primitive/FloatColumn.java
View File

@ -11,22 +11,29 @@ import java.nio.file.Path;
public class FloatColumn {
public static FloatColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(Storage.reader(path, columnDesc, true));
return new Reader(columnDesc, Storage.reader(path, columnDesc, true));
}
public static FloatColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(Storage.writer(path, columnDesc));
return new Writer(columnDesc, Storage.writer(path, columnDesc));
}
private static class Writer implements FloatColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private long position = 0;
public Writer(StorageWriter storageWriter) throws IOException {
public Writer(ColumnDesc<?, ?> columnDesc, StorageWriter storageWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(float value) throws IOException {
storage.putFloat(value);
position++;
@ -42,12 +49,19 @@ public class FloatColumn {
}
private static class Reader implements FloatColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader storage) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public float get() throws IOException {
return storage.getFloat();
}

24
code/libraries/slop/java/nu/marginalia/slop/column/primitive/IntColumn.java
View File

@ -11,21 +11,28 @@ import java.nio.file.Path;
public class IntColumn {
public static IntColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(Storage.reader(path, columnDesc, true));
return new Reader(columnDesc, Storage.reader(path, columnDesc, true));
}
public static IntColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(Storage.writer(path, columnDesc));
return new Writer(columnDesc, Storage.writer(path, columnDesc));
}
private static class Writer implements IntColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private long position = 0;
public Writer(StorageWriter storageWriter) throws IOException {
public Writer(ColumnDesc<?, ?> columnDesc, StorageWriter storageWriter) throws IOException {
this.columnDesc = columnDesc;
this.storage = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(int[] values) throws IOException {
for (int value : values) {
storage.putInt(value);
@ -39,7 +46,7 @@ public class IntColumn {
}
public long position() {
return position / Integer.BYTES;
return position;
}
public void close() throws IOException {
@ -48,12 +55,19 @@ public class IntColumn {
}
private static class Reader implements IntColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader storage) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public int get() throws IOException {
return storage.getInt();
}

22
code/libraries/slop/java/nu/marginalia/slop/column/primitive/LongColumn.java
View File

@ -11,21 +11,28 @@ import java.nio.file.Path;
public class LongColumn {
public static LongColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(Storage.reader(path, columnDesc, true));
return new Reader(columnDesc, Storage.reader(path, columnDesc, true));
}
public static LongColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(Storage.writer(path, columnDesc));
return new Writer(columnDesc, Storage.writer(path, columnDesc));
}
private static class Writer implements LongColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storage;
private long position = 0;
public Writer(StorageWriter storageWriter) {
public Writer(ColumnDesc<?, ?> columnDesc, StorageWriter storageWriter) {
this.columnDesc = columnDesc;
this.storage = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(long value) throws IOException {
storage.putLong(value);
position++;
@ -41,12 +48,19 @@ public class LongColumn {
}
private static class Reader implements LongColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storage;
public Reader(StorageReader storage) throws IOException {
public Reader(ColumnDesc<?, ?> columnDesc, StorageReader storage) throws IOException {
this.columnDesc = columnDesc;
this.storage = storage;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public long get() throws IOException {
return storage.getLong();
}

35
code/libraries/slop/java/nu/marginalia/slop/column/string/EnumColumn.java
View File

@ -16,16 +16,17 @@ import java.util.List;
public class EnumColumn {
public static StringColumnReader open(Path path, ColumnDesc name) throws IOException {
public static StringColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
return new Reader(
columnDesc,
StringColumn.open(path,
name.createSupplementaryColumn(
columnDesc.createSupplementaryColumn(
ColumnFunction.DICT,
ColumnType.TXTSTRING,
StorageType.PLAIN)
),
VarintColumn.open(path,
name.createSupplementaryColumn(
columnDesc.createSupplementaryColumn(
ColumnFunction.DATA,
ColumnType.ENUM_LE,
StorageType.PLAIN
@ -34,26 +35,34 @@ public class EnumColumn {
);
}
public static StringColumnWriter create(Path path, ColumnDesc name) throws IOException {
return new Writer(
StringColumn.create(path, name.createSupplementaryColumn(ColumnFunction.DICT, ColumnType.TXTSTRING, StorageType.PLAIN)),
VarintColumn.create(path, name.createSupplementaryColumn(ColumnFunction.DATA, ColumnType.ENUM_LE, StorageType.PLAIN))
public static StringColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
return new Writer(columnDesc,
StringColumn.create(path, columnDesc.createSupplementaryColumn(ColumnFunction.DICT, ColumnType.TXTSTRING, StorageType.PLAIN)),
VarintColumn.create(path, columnDesc.createSupplementaryColumn(ColumnFunction.DATA, ColumnType.ENUM_LE, StorageType.PLAIN))
);
}
private static class Writer implements StringColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StringColumnWriter dicionaryColumn;
private final LongColumnWriter dataColumn;
private final HashMap<String, Integer> dictionary = new HashMap<>();
public Writer(StringColumnWriter dicionaryColumn,
public Writer(ColumnDesc<?, ?> columnDesc,
StringColumnWriter dicionaryColumn,
LongColumnWriter dataColumn) throws IOException
{
this.columnDesc = columnDesc;
this.dicionaryColumn = dicionaryColumn;
this.dataColumn = dataColumn;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(String value) throws IOException {
Integer index = dictionary.get(value);
if (index == null) {
@ -75,12 +84,15 @@ public class EnumColumn {
}
private static class Reader implements StringColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final LongColumnReader dataColumn;
private final List<String> dictionary = new ArrayList<>();
public Reader(StringColumnReader dicionaryColumn,
public Reader(ColumnDesc<?, ?> columnDesc,
StringColumnReader dicionaryColumn,
LongColumnReader dataColumn) throws IOException
{
this.columnDesc = columnDesc;
this.dataColumn = dataColumn;
for (int i = 0; dicionaryColumn.hasRemaining(); i++) {
dictionary.add(dicionaryColumn.get());
@ -88,6 +100,11 @@ public class EnumColumn {
dicionaryColumn.close();
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public String get() throws IOException {
int index = (int) dataColumn.get();
return dictionary.get(index);

86
code/libraries/slop/java/nu/marginalia/slop/column/string/StringColumn.java
View File

@ -14,35 +14,42 @@ import java.nio.file.Path;
public class StringColumn {
public static StringColumnReader open(Path path, ColumnDesc name) throws IOException {
if (name.type().equals(ColumnType.STRING)) {
return new ArrayReader(ByteArrayColumn.open(path, name));
} else if (name.type().equals(ColumnType.CSTRING)) {
return new CStringReader(Storage.reader(path, name, true));
} else if (name.type().equals(ColumnType.TXTSTRING)) {
return new TxtStringReader(Storage.reader(path, name, true));
public static StringColumnReader open(Path path, ColumnDesc columnDesc) throws IOException {
if (columnDesc.type().equals(ColumnType.STRING)) {
return new ArrayReader(columnDesc, ByteArrayColumn.open(path, columnDesc));
} else if (columnDesc.type().equals(ColumnType.CSTRING)) {
return new CStringReader(columnDesc, Storage.reader(path, columnDesc, true));
} else if (columnDesc.type().equals(ColumnType.TXTSTRING)) {
return new TxtStringReader(columnDesc, Storage.reader(path, columnDesc, true));
}
throw new IllegalArgumentException("Unsupported column type: " + name.type());
throw new IllegalArgumentException("Unsupported column type: " + columnDesc.type());
}
public static StringColumnWriter create(Path path, ColumnDesc name) throws IOException {
if (name.type().equals(ColumnType.STRING)) {
return new ArrayWriter(ByteArrayColumn.create(path, name));
} else if (name.type().equals(ColumnType.CSTRING)) {
return new CStringWriter(Storage.writer(path, name));
} else if (name.type().equals(ColumnType.TXTSTRING)) {
return new TxtStringWriter(Storage.writer(path, name));
public static StringColumnWriter create(Path path, ColumnDesc columnDesc) throws IOException {
if (columnDesc.type().equals(ColumnType.STRING)) {
return new ArrayWriter(columnDesc, ByteArrayColumn.create(path, columnDesc));
} else if (columnDesc.type().equals(ColumnType.CSTRING)) {
return new CStringWriter(columnDesc, Storage.writer(path, columnDesc));
} else if (columnDesc.type().equals(ColumnType.TXTSTRING)) {
return new TxtStringWriter(columnDesc, Storage.writer(path, columnDesc));
}
throw new IllegalArgumentException("Unsupported column type: " + name.type());
throw new IllegalArgumentException("Unsupported column type: " + columnDesc.type());
}
private static class ArrayWriter implements StringColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final ByteArrayColumnWriter backingColumn;
public ArrayWriter(ByteArrayColumnWriter backingColumn) throws IOException {
public ArrayWriter(ColumnDesc<?, ?> columnDesc, ByteArrayColumnWriter backingColumn) throws IOException {
this.columnDesc = columnDesc;
this.backingColumn = backingColumn;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(String value) throws IOException {
if (null == value) {
value = "";
@ -61,12 +68,19 @@ public class StringColumn {
}
private static class ArrayReader implements StringColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final ByteArrayColumnReader backingColumn;
public ArrayReader(ByteArrayColumnReader backingColumn) throws IOException {
public ArrayReader(ColumnDesc<?, ?> columnDesc, ByteArrayColumnReader backingColumn) throws IOException {
this.columnDesc = columnDesc;
this.backingColumn = backingColumn;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public String get() throws IOException {
return new String(backingColumn.get());
}
@ -94,14 +108,21 @@ public class StringColumn {
private static class CStringWriter implements StringColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storageWriter;
private long position = 0;
public CStringWriter(StorageWriter storageWriter) throws IOException {
public CStringWriter(ColumnDesc<?,?> columnDesc, StorageWriter storageWriter) throws IOException {
this.columnDesc = columnDesc;
this.storageWriter = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(String value) throws IOException {
if (null == value) {
value = "";
@ -122,13 +143,20 @@ public class StringColumn {
}
private static class CStringReader implements StringColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storageReader;
private long position = 0;
public CStringReader(StorageReader storageReader) throws IOException {
public CStringReader(ColumnDesc<?, ?> columnDesc, StorageReader storageReader) throws IOException {
this.columnDesc = columnDesc;
this.storageReader = storageReader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public String get() throws IOException {
StringBuilder sb = new StringBuilder();
byte b;
@ -169,13 +197,20 @@ public class StringColumn {
private static class TxtStringWriter implements StringColumnWriter {
private final ColumnDesc<?, ?> columnDesc;
private final StorageWriter storageWriter;
private long position = 0;
public TxtStringWriter(StorageWriter storageWriter) throws IOException {
public TxtStringWriter(ColumnDesc<?, ?> columnDesc, StorageWriter storageWriter) throws IOException {
this.columnDesc = columnDesc;
this.storageWriter = storageWriter;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public void put(String value) throws IOException {
if (null == value) {
value = "";
@ -198,13 +233,20 @@ public class StringColumn {
}
private static class TxtStringReader implements StringColumnReader {
private final ColumnDesc<?, ?> columnDesc;
private final StorageReader storageReader;
private long position = 0;
public TxtStringReader(StorageReader storageReader) throws IOException {
public TxtStringReader(ColumnDesc<?, ?> columnDesc, StorageReader storageReader) throws IOException {
this.columnDesc = columnDesc;
this.storageReader = storageReader;
}
@Override
public ColumnDesc<?, ?> columnDesc() {
return columnDesc;
}
public String get() throws IOException {
StringBuilder sb = new StringBuilder();
byte b;

13
code/libraries/slop/java/nu/marginalia/slop/desc/SlopTable.java
View File

@ -51,14 +51,14 @@ public class SlopTable implements AutoCloseable {
public void close() throws IOException {
Set<Long> positions = new HashSet<>();
Map<Long, List<ColumnDesc>> positions = new HashMap<>();
for (ColumnReader reader : readerList) {
positions.add(reader.position());
positions.computeIfAbsent(reader.position(), k -> new ArrayList<>()).add(reader.columnDesc());
reader.close();
}
for (ColumnWriter writer : writerList) {
positions.add(writer.position());
positions.computeIfAbsent(writer.position(), k -> new ArrayList<>()).add(writer.columnDesc());
writer.close();
}
@ -68,14 +68,15 @@ public class SlopTable implements AutoCloseable {
// read or written to one of the columns. This is likely a bug,
// but not necessarily a severe one, so we just log a warning.
if (positions.remove(0L) && !positions.isEmpty()) {
logger.warn("Zero position found in one of the tables, this is likely development debris");
var zeroPositions = Objects.requireNonNullElseGet(positions.remove(0L), List::of);
if (!zeroPositions.isEmpty() && !positions.isEmpty()) {
logger.warn("Zero position found in {}, this is likely development debris", zeroPositions);
}
// If there are more than one position and several are non-zero, then we haven't maintained the
// position correctly between the columns. This is a disaster, so we throw an exception.
if (positions.size() > 1) {
throw new IllegalStateException("Expected only one reader position, was " + positions);
throw new IllegalStateException("Expected only one reader position, found " + positions);
}
for (var table : columnGroups.values()) {

215
code/libraries/slop/test/nu/marginalia/slop/desc/SlopTableTest.java Normal file
View File

@ -0,0 +1,215 @@
package nu.marginalia.slop.desc;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class SlopTableTest {
Path tempDir;
@BeforeEach
void setup() throws IOException {
tempDir = Files.createTempDirectory(getClass().getSimpleName());
}
@AfterEach
void cleanup() {
try {
Files.walk(tempDir)
.sorted(this::deleteOrder)
.forEach(p -> {
try {
if (Files.isRegularFile(p)) {
System.out.println("Deleting " + p + " " + Files.size(p));
}
Files.delete(p);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}
int deleteOrder(Path a, Path b) {
if (Files.isDirectory(a) && !Files.isDirectory(b)) {
return 1;
} else if (!Files.isDirectory(a) && Files.isDirectory(b)) {
return -1;
} else {
return a.getNameCount() - b.getNameCount();
}
}
@Test
public void testEmpty() throws IOException {
SlopTable slopTable = new SlopTable();
slopTable.close();
}
@Test
public void testPositionsGood() throws IOException {
var name1 = new ColumnDesc<>("test1",
0,
ColumnFunction.DATA,
ColumnType.INT_LE,
StorageType.PLAIN
);
var name2 = new ColumnDesc<>("test2",
0,
ColumnFunction.DATA,
ColumnType.INT_LE,
StorageType.PLAIN
);
try (SlopTable writerTable = new SlopTable()) {
var column1 = name1.create(writerTable, tempDir);
var column2 = name2.create(writerTable, tempDir);
column1.put(42);
column2.put(43);
}
try (SlopTable readerTable = new SlopTable()) {
var column1 = name1.open(readerTable, tempDir);
var column2 = name2.open(readerTable, tempDir);
assertEquals(42, column1.get());
assertEquals(43, column2.get());
}
}
@Test
public void testPositionsMisaligned() throws IOException {
var name1 = new ColumnDesc<>("test1",
0,
ColumnFunction.DATA,
ColumnType.INT_LE,
StorageType.PLAIN
);
var name2 = new ColumnDesc<>("test2",
0,
ColumnFunction.DATA,
ColumnType.INT_LE,
StorageType.PLAIN
);
boolean sawException = false;
try (SlopTable writerTable = new SlopTable()) {
var column1 = name1.create(writerTable, tempDir);
var column2 = name2.create(writerTable, tempDir);
column1.put(42);
column2.put(43);
column2.put(44);
}
catch (Exception ex) {
ex.printStackTrace();
sawException = true;
}
assertEquals(true, sawException);
}
// Sanity check for the implementation of position() in the column classes
@Test
public void testPositionsMegatest() throws IOException {
var byteCol = new ColumnDesc<>("byte", ColumnType.BYTE, StorageType.PLAIN);
var charCol = new ColumnDesc<>("char", ColumnType.CHAR_LE, StorageType.PLAIN);
var intCol = new ColumnDesc<>("int", ColumnType.INT_LE, StorageType.PLAIN);
var longCol = new ColumnDesc<>("long", ColumnType.LONG_LE, StorageType.PLAIN);
var floatCol = new ColumnDesc<>("float", ColumnType.FLOAT_LE, StorageType.PLAIN);
var doubleCol = new ColumnDesc<>("double", ColumnType.DOUBLE_LE, StorageType.PLAIN);
var byteArrayCol = new ColumnDesc<>("byteArray", ColumnType.BYTE_ARRAY, StorageType.PLAIN);
var intArrayCol = new ColumnDesc<>("intArray", ColumnType.INT_ARRAY_LE, StorageType.PLAIN);
var longArrayCol = new ColumnDesc<>("longArray", ColumnType.LONG_ARRAY_LE, StorageType.PLAIN);
var cstringCol = new ColumnDesc<>("cstring", ColumnType.CSTRING, StorageType.PLAIN);
var txtStringCol = new ColumnDesc<>("txtString", ColumnType.TXTSTRING, StorageType.PLAIN);
var arrayStringCol = new ColumnDesc<>("arrayString", ColumnType.STRING, StorageType.PLAIN);
var varintCol = new ColumnDesc<>("varint", ColumnType.VARINT_LE, StorageType.PLAIN);
var enumCol = new ColumnDesc<>("enum", ColumnType.ENUM_LE, StorageType.PLAIN);
try (SlopTable writerTable = new SlopTable()) {
var byteColumn = byteCol.create(writerTable, tempDir);
var charColumn = charCol.create(writerTable, tempDir);
var intColumn = intCol.create(writerTable, tempDir);
var longColumn = longCol.create(writerTable, tempDir);
var floatColumn = floatCol.create(writerTable, tempDir);
var doubleColumn = doubleCol.create(writerTable, tempDir);
var byteArrayColumn = byteArrayCol.create(writerTable, tempDir);
var intArrayColumn = intArrayCol.create(writerTable, tempDir);
var longArrayColumn = longArrayCol.create(writerTable, tempDir);
var cstringColumn = cstringCol.create(writerTable, tempDir);
var txtStringColumn = txtStringCol.create(writerTable, tempDir);
var arrayStringColumn = arrayStringCol.create(writerTable, tempDir);
var enumColumn = enumCol.create(writerTable, tempDir);
var varintColumn = varintCol.create(writerTable, tempDir);
byteColumn.put((byte) 42);
charColumn.put('a');
intColumn.put(42);
longColumn.put(42L);
floatColumn.put(42.0f);
doubleColumn.put(42.0);
byteArrayColumn.put(new byte[] { 42, 43, 44 });
intArrayColumn.put(new int[] { 42, 43, 44 });
longArrayColumn.put(new long[] { 42, 43, 44 });
cstringColumn.put("Hello");
txtStringColumn.put("Hello");
arrayStringColumn.put("Hello");
enumColumn.put("Hello");
varintColumn.put(10000000);
}
try (SlopTable readerTable = new SlopTable()) {
var byteColumn = byteCol.open(readerTable, tempDir);
var charColumn = charCol.open(readerTable, tempDir);
var intColumn = intCol.open(readerTable, tempDir);
var longColumn = longCol.open(readerTable, tempDir);
var floatColumn = floatCol.open(readerTable, tempDir);
var doubleColumn = doubleCol.open(readerTable, tempDir);
var byteArrayColumn = byteArrayCol.open(readerTable, tempDir);
var intArrayColumn = intArrayCol.open(readerTable, tempDir);
var longArrayColumn = longArrayCol.open(readerTable, tempDir);
var cstringColumn = cstringCol.open(readerTable, tempDir);
var txtStringColumn = txtStringCol.open(readerTable, tempDir);
var arrayStringColumn = arrayStringCol.open(readerTable, tempDir);
var enumColumn = enumCol.open(readerTable, tempDir);
var varintColumn = varintCol.open(readerTable, tempDir);
assertEquals(42, byteColumn.get());
assertEquals('a', charColumn.get());
assertEquals(42, intColumn.get());
assertEquals(42L, longColumn.get());
assertEquals(42.0f, floatColumn.get());
assertEquals(42.0, doubleColumn.get());
assertArrayEquals(new byte[] {42, 43, 44}, byteArrayColumn.get());
assertArrayEquals(new int[] {42, 43, 44}, intArrayColumn.get());
assertArrayEquals(new long[] {42, 43, 44}, longArrayColumn.get());
assertEquals("Hello", cstringColumn.get());
assertEquals("Hello", txtStringColumn.get());
assertEquals("Hello", arrayStringColumn.get());
assertEquals("Hello", enumColumn.get());
assertEquals(10000000, varintColumn.get());
}
}
}