diff --git a/marginalia_nu/build.gradle b/marginalia_nu/build.gradle index e1c96de0..638c1e30 100644 --- a/marginalia_nu/build.gradle +++ b/marginalia_nu/build.gradle @@ -106,8 +106,9 @@ dependencies { implementation group: 'org.yaml', name: 'snakeyaml', version: '1.30' - implementation 'com.syncthemall:boilerpipe:1.2.2' implementation 'com.github.luben:zstd-jni:1.5.2-2' + implementation 'org.lz4:lz4-java:1.8.0' + implementation 'com.github.vladimir-bukhtoyarov:bucket4j-core:7.5.0' implementation 'de.rototor.jeuclid:jeuclid-core:3.1.14' @@ -126,7 +127,6 @@ dependencies { implementation 'org.roaringbitmap:RoaringBitmap:0.9.32' implementation group: 'mysql', name: 'mysql-connector-java', version: '8.0.29' - implementation 'com.github.Marcono1234:gson-record-type-adapter-factory:0.2.0' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/BigString.java b/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/BigString.java new file mode 100644 index 00000000..48c4c053 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/BigString.java @@ -0,0 +1,17 @@ +package nu.marginalia.util.bigstring; + +public interface BigString { + static BigString encode(String stringValue) { + if (stringValue.length() > 64) { + return new CompressedBigString(stringValue); + } + else { + return new PlainBigString(stringValue); + } + } + String decode(); + + byte[] getBytes(); + + int length(); +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/CompressedBigString.java b/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/CompressedBigString.java new file mode 100644 index 00000000..1b84e576 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/CompressedBigString.java @@ -0,0 +1,39 @@ +package nu.marginalia.util.bigstring; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + +import java.nio.charset.StandardCharsets; + +public class CompressedBigString implements BigString { + private final int originalSize; + private final int length; + private final byte[] encoded; + + private static final LZ4Factory lz4Factory = LZ4Factory.fastestInstance();; + private static final LZ4Compressor compressor = lz4Factory.fastCompressor(); + private static final LZ4FastDecompressor decompressor = lz4Factory.fastDecompressor(); + + public CompressedBigString(String stringValue) { + byte[] byteValue = stringValue.getBytes(StandardCharsets.UTF_16); + originalSize = byteValue.length; + encoded = compressor.compress(byteValue); + length = stringValue.length(); + } + + @Override + public String decode() { + return new String(getBytes(), StandardCharsets.UTF_16); + } + + @Override + public byte[] getBytes() { + return decompressor.decompress(encoded, originalSize); + } + + @Override + public int length() { + return length; + } +} diff --git a/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/PlainBigString.java b/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/PlainBigString.java new file mode 100644 index 00000000..5af3a5c8 --- /dev/null +++ b/marginalia_nu/src/main/java/nu/marginalia/util/bigstring/PlainBigString.java @@ -0,0 +1,26 @@ +package nu.marginalia.util.bigstring; + +import java.nio.charset.StandardCharsets; + +public class PlainBigString implements BigString { + private final String value; + + public PlainBigString(String value) { + this.value = value; + } + + @Override + public String decode() { + return value; + } + + @Override + public byte[] getBytes() { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public int length() { + return value.length(); + } +}