Improvements to array library and conversion

This commit is contained in:
Viktor Lofgren 2023-02-02 10:35:14 +01:00
parent cdaeb7724a
commit b18cd0bc36
4 changed files with 23 additions and 3 deletions

View File

@ -108,7 +108,6 @@ public class RandomWriteFunnel implements AutoCloseable {
private void eval(ByteBuffer dest) throws IOException { private void eval(ByteBuffer dest) throws IOException {
flushBuffer(); flushBuffer();
channel.force(false);
channel.position(0); channel.position(0);
buffer.clear(); buffer.clear();

View File

@ -62,6 +62,16 @@ public class ShiftedIntArray implements IntArray {
delegate.get(shift+start, shift+end, buffer); delegate.get(shift+start, shift+end, buffer);
} }
@Override
public int getAndIncrement(long pos) {
return delegate.getAndIncrement(shift + pos);
}
@Override
public void fill(long start, long end, int val) {
delegate.fill(start + shift, end + shift, val);
}
@Override @Override
public long size() { public long size() {
return size; return size;

View File

@ -63,6 +63,16 @@ public class ShiftedLongArray implements LongArray {
delegate.get(shift+start, shift+end, buffer); delegate.get(shift+start, shift+end, buffer);
} }
@Override
public long getAndIncrement(long pos) {
return delegate.getAndIncrement(shift + pos);
}
@Override
public void fill(long start, long end, long val) {
delegate.fill(start + shift, end + shift, val);
}
@Override @Override
public long size() { public long size() {
return size; return size;

View File

@ -34,7 +34,7 @@ public class ReverseIndexConverter {
private final SearchIndexJournalReaderSingleFile journalReader; private final SearchIndexJournalReaderSingleFile journalReader;
private final Path outputFileWords; private final Path outputFileWords;
private final Path outputFileDocs; private final Path outputFileDocs;
private final SortingContext sortingContext;
public ReverseIndexConverter(Path tmpFileDir, public ReverseIndexConverter(Path tmpFileDir,
SearchIndexJournalReaderSingleFile journalReader, SearchIndexJournalReaderSingleFile journalReader,
@ -44,6 +44,7 @@ public class ReverseIndexConverter {
this.journalReader = journalReader; this.journalReader = journalReader;
this.outputFileWords = outputFileWords; this.outputFileWords = outputFileWords;
this.outputFileDocs = outputFileDocs; this.outputFileDocs = outputFileDocs;
this.sortingContext = new SortingContext(tmpFileDir, 64_000);
} }
public void convert() throws IOException { public void convert() throws IOException {
@ -56,7 +57,7 @@ public class ReverseIndexConverter {
final SearchIndexJournalStatistics statistics = journalReader.getStatistics(); final SearchIndexJournalStatistics statistics = journalReader.getStatistics();
final Path intermediateUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat"); final Path intermediateUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat");
SortingContext sortingContext = new SortingContext(tmpFileDir, 64_000);
try { try {
final long wordsFileSize = statistics.highestWord() + 1; final long wordsFileSize = statistics.highestWord() + 1;