Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(coded-sequence) Add a callback for re-filling underlying buffer

Browse Source
This commit is contained in:
Viktor Lofgren 2024-07-12 23:50:28 +02:00
parent d28fc86956
commit 179a6002c2
2 changed files with 26 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
code/index/index-reverse/java/nu/marginalia/index/PrioIndexEntrySource.java
View File

@ -15,7 +15,7 @@ public class PrioIndexEntrySource implements EntrySource {
private final String name; private final String name;
private final ByteBuffer readData = ByteBuffer.allocate(1024); private final ByteBuffer readData = ByteBuffer.allocate(1024);
private final BitReader bitReader = new BitReader(readData); private final BitReader bitReader = new BitReader(readData, this::fillReadBuffer);
private final FileChannel docsFileChannel; private final FileChannel docsFileChannel;
private long dataOffsetStartB; private long dataOffsetStartB;
@ -69,8 +69,6 @@ public class PrioIndexEntrySource implements EntrySource {
outputBuffer.clear(); outputBuffer.clear();
while (outputBuffer.hasRemaining() && readItems++ < numItems) { while (outputBuffer.hasRemaining() && readItems++ < numItems) {
fillReadBuffer();
int rank; int rank;
int domainId; int domainId;
int docOrd; int docOrd;
@ -119,8 +117,8 @@ public class PrioIndexEntrySource implements EntrySource {
buffer.uniq(); buffer.uniq();
} }
private void fillReadBuffer() throws IOException { private void fillReadBuffer() {
if (readData.remaining() < 8) { try {
readData.compact(); readData.compact();
int rb = docsFileChannel.read(readData, dataOffsetStartB); int rb = docsFileChannel.read(readData, dataOffsetStartB);
if (rb > 0) { if (rb > 0) {
@ -128,6 +126,9 @@ public class PrioIndexEntrySource implements EntrySource {
} }
readData.flip(); readData.flip();
} }
catch (IOException ex) {
throw new IllegalStateException("Failed to read index data.", ex);
}
} }
@Override @Override

22
code/libraries/coded-sequence/java/nu/marginalia/sequence/io/BitReader.java
View File

@ -1,5 +1,8 @@
package nu.marginalia.sequence.io; package nu.marginalia.sequence.io;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
/** A utility class for reading bits from a ByteBuffer /** A utility class for reading bits from a ByteBuffer
@ -7,6 +10,9 @@ import java.nio.ByteBuffer;
*/ */
public class BitReader { public class BitReader {
private final ByteBuffer underlying; private final ByteBuffer underlying;
private final Runnable refillCallback;
private static final Logger logger = LoggerFactory.getLogger(BitReader.class);
/** The current value being decoded */ /** The current value being decoded */
private long currentValue; private long currentValue;
@ -14,12 +20,23 @@ public class BitReader {
/** Bit index in the current value */ /** Bit index in the current value */
private int bitPosition; private int bitPosition;
public BitReader(ByteBuffer buffer) {
/** Create a new BitReader for the given buffer. The supplied callback will be
* invoked when the underlying buffer is out of data. The callback should
* refill the buffer with more data.
*/
public BitReader(ByteBuffer buffer, Runnable refillCallback) {
this.underlying = buffer; this.underlying = buffer;
this.refillCallback = refillCallback;
this.bitPosition = 0; this.bitPosition = 0;
this.currentValue = 0; this.currentValue = 0;
} }
/** Create a new BitReader for the given buffer */
public BitReader(ByteBuffer buffer) {
this(buffer, () -> { throw new IllegalStateException("No more data to read and no re-fill callback provided"); });
}
/** Read the next bit from the buffer */ /** Read the next bit from the buffer */
public boolean getBit() { public boolean getBit() {
if (bitPosition <= 0) { if (bitPosition <= 0) {
@ -132,7 +149,8 @@ public class BitReader {
bitPosition = 8; bitPosition = 8;
} }
else { // There's no more data to read! else { // There's no more data to read!
throw new ArrayIndexOutOfBoundsException("No more data to read"); refillCallback.run();
readNext();
} }
} }
} }