Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Improvements to array library and conversion

Browse Source
This commit is contained in:
Viktor Lofgren 2023-02-02 10:35:14 +01:00
parent cdaeb7724a
commit b18cd0bc36
4 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
marginalia_nu/src/main/java/nu/marginalia/util/RandomWriteFunnel.java
View File

@ -108,7 +108,6 @@ public class RandomWriteFunnel implements AutoCloseable {
private void eval(ByteBuffer dest) throws IOException {
flushBuffer();
channel.force(false);
channel.position(0);
buffer.clear();

10
marginalia_nu/src/main/java/nu/marginalia/util/array/delegate/ShiftedIntArray.java
View File

@ -62,6 +62,16 @@ public class ShiftedIntArray implements IntArray {
delegate.get(shift+start, shift+end, buffer);
}
@Override
public int getAndIncrement(long pos) {
return delegate.getAndIncrement(shift + pos);
}
@Override
public void fill(long start, long end, int val) {
delegate.fill(start + shift, end + shift, val);
}
@Override
public long size() {
return size;

10
marginalia_nu/src/main/java/nu/marginalia/util/array/delegate/ShiftedLongArray.java
View File

@ -63,6 +63,16 @@ public class ShiftedLongArray implements LongArray {
delegate.get(shift+start, shift+end, buffer);
}
@Override
public long getAndIncrement(long pos) {
return delegate.getAndIncrement(shift + pos);
}
@Override
public void fill(long start, long end, long val) {
delegate.fill(start + shift, end + shift, val);
}
@Override
public long size() {
return size;

5
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/postings/reverse/ReverseIndexConverter.java
View File

@ -34,7 +34,7 @@ public class ReverseIndexConverter {
private final SearchIndexJournalReaderSingleFile journalReader;
private final Path outputFileWords;
private final Path outputFileDocs;
private final SortingContext sortingContext;
public ReverseIndexConverter(Path tmpFileDir,
SearchIndexJournalReaderSingleFile journalReader,
@ -44,6 +44,7 @@ public class ReverseIndexConverter {
this.journalReader = journalReader;
this.outputFileWords = outputFileWords;
this.outputFileDocs = outputFileDocs;
this.sortingContext = new SortingContext(tmpFileDir, 64_000);
}
public void convert() throws IOException {
@ -56,7 +57,7 @@ public class ReverseIndexConverter {
final SearchIndexJournalStatistics statistics = journalReader.getStatistics();
final Path intermediateUrlsFile = Files.createTempFile(tmpFileDir, "urls-sorted", ".dat");
SortingContext sortingContext = new SortingContext(tmpFileDir, 64_000);
try {
final long wordsFileSize = statistics.highestWord() + 1;