diff --git a/run/reconvert.sh b/run/reconvert.sh index 468ff2b3..608d7448 100755 --- a/run/reconvert.sh +++ b/run/reconvert.sh @@ -85,5 +85,6 @@ converter-process ${SAMPLE_DIR}/plan.yaml loader-process ${SAMPLE_DIR}/plan.yaml mv vol/iw/index.dat vol/iw/0/page-index.dat +rm -f vol/ir/0/* popd diff --git a/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java index c6552c18..4b4476e9 100644 --- a/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java +++ b/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java @@ -7,26 +7,34 @@ import io.reactivex.rxjava3.schedulers.Schedulers; import nu.marginalia.index.index.SearchIndex; import nu.marginalia.index.svc.IndexOpsService; import nu.marginalia.index.svc.IndexQueryService; +import nu.marginalia.index.svc.IndexSearchSetsService; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.service.server.Initialization; import nu.marginalia.service.server.MetricsServer; import nu.marginalia.service.server.Service; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; import spark.Spark; +import java.io.IOException; import java.util.concurrent.TimeUnit; import static spark.Spark.get; public class IndexService extends Service { + private final Logger logger = LoggerFactory.getLogger(getClass()); @NotNull private final Initialization init; private final IndexOpsService opsService; private final SearchIndex searchIndex; + private final IndexServicesFactory servicesFactory; + private final IndexSearchSetsService searchSetsService; + @Inject public IndexService(@Named("service-host") String ip, @@ -35,12 +43,15 @@ public class IndexService extends Service { MetricsServer metricsServer, IndexOpsService opsService, IndexQueryService indexQueryService, - SearchIndex searchIndex - ) + SearchIndex searchIndex, + IndexServicesFactory servicesFactory, + IndexSearchSetsService searchSetsService) { super(ip, port, init, metricsServer); this.opsService = opsService; this.searchIndex = searchIndex; + this.servicesFactory = servicesFactory; + this.searchSetsService = searchSetsService; final Gson gson = GsonFactory.get(); @@ -61,6 +72,7 @@ public class IndexService extends Service { } volatile boolean initialized = false; + public void initialize() { if (!initialized) { init.waitReady(); @@ -68,6 +80,22 @@ public class IndexService extends Service { initialized = true; } + if (!opsService.run(this::autoConvert)) { + logger.warn("Auto-convert could not be performed, ops lock busy"); + } + } + + private void autoConvert() { + if (!servicesFactory.isConvertedIndexMissing() || !servicesFactory.isPreconvertedIndexPresent()) { + return; + } + try { + searchSetsService.recalculateAll(); + searchIndex.switchIndex(); + } + catch (IOException ex) { + logger.error("Auto convert failed", ex); + } } diff --git a/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java b/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java index 9b33cb8a..05de1c08 100644 --- a/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java +++ b/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.List; import java.util.concurrent.Callable; +import java.util.stream.Stream; @Singleton public class IndexServicesFactory { @@ -42,6 +44,7 @@ public class IndexServicesFactory { int LIVE_PART = 0; int NEXT_PART = 1; + @Inject public IndexServicesFactory( @Named("tmp-file-dir") Path tmpFileDir, @@ -72,6 +75,28 @@ public class IndexServicesFactory { return searchSetsBase; } + public boolean isPreconvertedIndexPresent() { + return Stream.of( + revIndexWords.get(NEXT_PART).toPath(), + revIndexDoc.get(NEXT_PART).toPath(), + revPrioIndexWords.get(NEXT_PART).toPath(), + revPrioIndexDoc.get(NEXT_PART).toPath(), + fwdIndexDocData.get(NEXT_PART).toPath(), + fwdIndexDocId.get(NEXT_PART).toPath() + ).allMatch(Files::exists); + } + + public boolean isConvertedIndexMissing() { + return !Stream.of( + revIndexWords.get(LIVE_PART).toPath(), + revIndexDoc.get(LIVE_PART).toPath(), + revPrioIndexWords.get(LIVE_PART).toPath(), + revPrioIndexDoc.get(LIVE_PART).toPath(), + fwdIndexDocData.get(LIVE_PART).toPath(), + fwdIndexDocId.get(LIVE_PART).toPath() + ).allMatch(Files::exists); + } + public void convertIndex(DomainRankings domainRankings) throws IOException { convertForwardIndex(domainRankings); convertFullReverseIndex(domainRankings); diff --git a/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java b/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java index b5b4677c..b6be4ef4 100644 --- a/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java +++ b/services-core/index-service/src/main/java/nu/marginalia/index/index/SearchIndex.java @@ -31,7 +31,7 @@ public class SearchIndex { @NotNull private final IndexServicesFactory servicesFactory; - private IndexSearchSetsService searchSetsService; + private final IndexSearchSetsService searchSetsService; @Inject public SearchIndex(@NotNull IndexServicesFactory servicesFactory, IndexSearchSetsService searchSetsService) { diff --git a/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java b/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java index 36561dc7..34ed2927 100644 --- a/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java +++ b/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java @@ -48,7 +48,7 @@ public class IndexOpsService { @CheckReturnValue - private Optional run(Callable c) throws Exception { + public Optional run(Callable c) throws Exception { if (!opsLock.tryLock()) return Optional.empty(); try { @@ -61,7 +61,7 @@ public class IndexOpsService { @CheckReturnValue - private boolean run(Runnable r) { + public boolean run(Runnable r) { if (!opsLock.tryLock()) return false; try {