Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Instrumentation for search + index madvise tweaks (#17)

Browse Source 
Co-authored-by: vlofgren <vlofgren@gmail.com>
Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/17
This commit is contained in:
Viktor Lofgren 2022-05-30 21:02:53 +02:00
parent 275e42197c
commit c201201c2d
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/service/index/SearchIndex.java
View File

@ -3,15 +3,18 @@ package nu.marginalia.wmsa.edge.index.service.index;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.google.inject.name.Named; import com.google.inject.name.Named;
import com.upserve.uppend.blobs.NativeIO; import com.upserve.uppend.blobs.NativeIO;
import io.reactivex.rxjava3.schedulers.Schedulers;
import nu.marginalia.wmsa.edge.index.service.index.wordstable.IndexWordsTable; import nu.marginalia.wmsa.edge.index.service.index.wordstable.IndexWordsTable;
import nu.marginalia.util.btree.BTreeReader; import nu.marginalia.util.btree.BTreeReader;
import nu.marginalia.util.multimap.MultimapFileLong; import nu.marginalia.util.multimap.MultimapFileLong;
import org.eclipse.jetty.util.thread.ThreadPool;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.util.concurrent.ForkJoinPool;
import java.util.stream.LongStream; import java.util.stream.LongStream;
public class SearchIndex implements AutoCloseable { public class SearchIndex implements AutoCloseable {
@ -40,12 +43,12 @@ public class SearchIndex implements AutoCloseable {
bTreeReader = new BTreeReader(urls, SearchIndexConverter.urlsBTreeContext); bTreeReader = new BTreeReader(urls, SearchIndexConverter.urlsBTreeContext);
madvise(urls, bTreeReader); Schedulers.io().scheduleDirect(() -> madvise(urls, bTreeReader));
} }
private void madvise(MultimapFileLong urls, BTreeReader reader) { private void madvise(MultimapFileLong urls, BTreeReader reader) {
urls.advice(NativeIO.Advice.Sequential); urls.advice(NativeIO.Advice.Random);
words.forEachWordsOffset(offset -> { words.forEachWordsOffset(offset -> {
var h = reader.getHeader(offset); var h = reader.getHeader(offset);
int length = (int) (h.dataOffsetLongs() - h.indexOffsetLongs()); int length = (int) (h.dataOffsetLongs() - h.indexOffsetLongs());