Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

WIP run and setup

Browse Source
This commit is contained in:
Viktor Lofgren 2023-03-04 15:41:54 +01:00
parent ef87e123ba
commit 5f758cbb0e
5 changed files with 59 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
run/reconvert.sh
View File

@ -85,5 +85,6 @@ converter-process ${SAMPLE_DIR}/plan.yaml
loader-process ${SAMPLE_DIR}/plan.yaml
mv vol/iw/index.dat vol/iw/0/page-index.dat
rm -f vol/ir/0/*
popd

32
services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java
View File

@ -7,26 +7,34 @@ import io.reactivex.rxjava3.schedulers.Schedulers;
import nu.marginalia.index.index.SearchIndex;
import nu.marginalia.index.svc.IndexOpsService;
import nu.marginalia.index.svc.IndexQueryService;
import nu.marginalia.index.svc.IndexSearchSetsService;
import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.service.server.Initialization;
import nu.marginalia.service.server.MetricsServer;
import nu.marginalia.service.server.Service;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.Request;
import spark.Response;
import spark.Spark;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import static spark.Spark.get;
public class IndexService extends Service {
private final Logger logger = LoggerFactory.getLogger(getClass());
@NotNull
private final Initialization init;
private final IndexOpsService opsService;
private final SearchIndex searchIndex;
private final IndexServicesFactory servicesFactory;
private final IndexSearchSetsService searchSetsService;
@Inject
public IndexService(@Named("service-host") String ip,
@ -35,12 +43,15 @@ public class IndexService extends Service {
MetricsServer metricsServer,
IndexOpsService opsService,
IndexQueryService indexQueryService,
SearchIndex searchIndex
)
SearchIndex searchIndex,
IndexServicesFactory servicesFactory,
IndexSearchSetsService searchSetsService)
{
super(ip, port, init, metricsServer);
this.opsService = opsService;
this.searchIndex = searchIndex;
this.servicesFactory = servicesFactory;
this.searchSetsService = searchSetsService;
final Gson gson = GsonFactory.get();
@ -61,6 +72,7 @@ public class IndexService extends Service {
}
volatile boolean initialized = false;
public void initialize() {
if (!initialized) {
init.waitReady();
@ -68,6 +80,22 @@ public class IndexService extends Service {
initialized = true;
}
if (!opsService.run(this::autoConvert)) {
logger.warn("Auto-convert could not be performed, ops lock busy");
}
}
private void autoConvert() {
if (!servicesFactory.isConvertedIndexMissing() || !servicesFactory.isPreconvertedIndexPresent()) {
return;
}
try {
searchSetsService.recalculateAll();
searchIndex.switchIndex();
}
catch (IOException ex) {
logger.error("Auto convert failed", ex);
}
}

25
services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java
View File

@ -20,7 +20,9 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.stream.Stream;
@Singleton
public class IndexServicesFactory {
@ -42,6 +44,7 @@ public class IndexServicesFactory {
int LIVE_PART = 0;
int NEXT_PART = 1;
@Inject
public IndexServicesFactory(
@Named("tmp-file-dir") Path tmpFileDir,
@ -72,6 +75,28 @@ public class IndexServicesFactory {
return searchSetsBase;
}
public boolean isPreconvertedIndexPresent() {
return Stream.of(
revIndexWords.get(NEXT_PART).toPath(),
revIndexDoc.get(NEXT_PART).toPath(),
revPrioIndexWords.get(NEXT_PART).toPath(),
revPrioIndexDoc.get(NEXT_PART).toPath(),
fwdIndexDocData.get(NEXT_PART).toPath(),
fwdIndexDocId.get(NEXT_PART).toPath()
).allMatch(Files::exists);
}
public boolean isConvertedIndexMissing() {
return !Stream.of(
revIndexWords.get(LIVE_PART).toPath(),
revIndexDoc.get(LIVE_PART).toPath(),
revPrioIndexWords.get(LIVE_PART).toPath(),
revPrioIndexDoc.get(LIVE_PART).toPath(),
fwdIndexDocData.get(LIVE_PART).toPath(),
fwdIndexDocId.get(LIVE_PART).toPath()
).allMatch(Files::exists);
}
public void convertIndex(DomainRankings domainRankings) throws IOException {
convertForwardIndex(domainRankings);
convertFullReverseIndex(domainRankings);

2
services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java
View File

@ -31,7 +31,7 @@ public class SearchIndex {
@NotNull
private final IndexServicesFactory servicesFactory;
private IndexSearchSetsService searchSetsService;
private final IndexSearchSetsService searchSetsService;
@Inject
public SearchIndex(@NotNull IndexServicesFactory servicesFactory, IndexSearchSetsService searchSetsService) {

4
services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java
View File

@ -48,7 +48,7 @@ public class IndexOpsService {
@CheckReturnValue
private <T> Optional<T> run(Callable<T> c) throws Exception {
public <T> Optional<T> run(Callable<T> c) throws Exception {
if (!opsLock.tryLock())
return Optional.empty();
try {
@ -61,7 +61,7 @@ public class IndexOpsService {
@CheckReturnValue
private boolean run(Runnable r) {
public boolean run(Runnable r) {
if (!opsLock.tryLock())
return false;
try {