mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-02-24 05:18:58 +00:00
Improvements to array library and conversion
This commit is contained in:
parent
cdaeb7724a
commit
b18cd0bc36
@ -108,7 +108,6 @@ public class RandomWriteFunnel implements AutoCloseable {
|
|||||||
|
|
||||||
private void eval(ByteBuffer dest) throws IOException {
|
private void eval(ByteBuffer dest) throws IOException {
|
||||||
flushBuffer();
|
flushBuffer();
|
||||||
channel.force(false);
|
|
||||||
|
|
||||||
channel.position(0);
|
channel.position(0);
|
||||||
buffer.clear();
|
buffer.clear();
|
||||||
|
@ -62,6 +62,16 @@ public class ShiftedIntArray implements IntArray {
|
|||||||
delegate.get(shift+start, shift+end, buffer);
|
delegate.get(shift+start, shift+end, buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getAndIncrement(long pos) {
|
||||||
|
return delegate.getAndIncrement(shift + pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void fill(long start, long end, int val) {
|
||||||
|
delegate.fill(start + shift, end + shift, val);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long size() {
|
public long size() {
|
||||||
return size;
|
return size;
|
||||||
|
@ -63,6 +63,16 @@ public class ShiftedLongArray implements LongArray {
|
|||||||
delegate.get(shift+start, shift+end, buffer);
|
delegate.get(shift+start, shift+end, buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getAndIncrement(long pos) {
|
||||||
|
return delegate.getAndIncrement(shift + pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void fill(long start, long end, long val) {
|
||||||
|
delegate.fill(start + shift, end + shift, val);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long size() {
|
public long size() {
|
||||||
return size;
|
return size;
|
||||||
|
@ -34,7 +34,7 @@ public class ReverseIndexConverter {
|
|||||||
private final SearchIndexJournalReaderSingleFile journalReader;
|
private final SearchIndexJournalReaderSingleFile journalReader;
|
||||||
private final Path outputFileWords;
|
private final Path outputFileWords;
|
||||||
private final Path outputFileDocs;
|
private final Path outputFileDocs;
|
||||||
|
private final SortingContext sortingContext;
|
||||||
|
|
||||||
public ReverseIndexConverter(Path tmpFileDir,
|
public ReverseIndexConverter(Path tmpFileDir,
|
||||||
SearchIndexJournalReaderSingleFile journalReader,
|
SearchIndexJournalReaderSingleFile journalReader,
|
||||||
@ -44,6 +44,7 @@ public class ReverseIndexConverter {
|
|||||||
this.journalReader = journalReader;
|
this.journalReader = journalReader;
|
||||||
this.outputFileWords = outputFileWords;
|
this.outputFileWords = outputFileWords;
|
||||||
this.outputFileDocs = outputFileDocs;
|
this.outputFileDocs = outputFileDocs;
|
||||||
|
this.sortingContext = new SortingContext(tmpFileDir, 64_000);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void convert() throws IOException {
|
public void convert() throws IOException {
|
||||||
@ -56,7 +57,7 @@ public class ReverseIndexConverter {
|
|||||||
final SearchIndexJournalStatistics statistics = journalReader.getStatistics();
|
final SearchIndexJournalStatistics statistics = journalReader.getStatistics();
|
||||||
|
|
||||||
final Path intermediateUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat");
|
final Path intermediateUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat");
|
||||||
SortingContext sortingContext = new SortingContext(tmpFileDir, 64_000);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
final long wordsFileSize = statistics.highestWord() + 1;
|
final long wordsFileSize = statistics.highestWord() + 1;
|
||||||
|
Loading…
Reference in New Issue
Block a user