diff --git a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java index f8349eb7..627ffd4f 100644 --- a/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java +++ b/code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java @@ -3,8 +3,6 @@ package nu.marginalia.index.client; public class IndexMqEndpoints { public static final String INDEX_IS_BLOCKED = "INDEX-IS-BLOCKED"; public static final String INDEX_REPARTITION = "INDEX-REPARTITION"; - - public static final String INDEX_RELOAD_LEXICON = "INDEX-RELOAD-LEXICON"; - public static final String INDEX_REINDEX = "INDEX-REINDEX"; + public static final String SWITCH_INDEX = "SWITCH-INDEX"; } diff --git a/code/processes/readme.md b/code/processes/readme.md index 44c29a1e..6b1ccede 100644 --- a/code/processes/readme.md +++ b/code/processes/readme.md @@ -19,6 +19,11 @@ described in [converting-model](../process-models/converting-model/). The [loading-process](loading-process/) reads the processed data and creates an index journal and lexicon, and loads domains and addresses into the MariaDB-database. +## 4. Index Construction Process + +The [index-construction-process](index-constructor-process/) constructs indices from +the data generated by the loader. + ## Overview Schematically the crawling and loading process looks like this: @@ -65,8 +70,12 @@ Schematically the crawling and loading process looks like this: \\==================// | +------------+ - | LOADING | Insert URLs in DB + | LOADING | Insert URLs in link DB | STEP | Insert keywords in Index +------------+ - + | + +------------+ + | CONSTRUCT | Make the data searchable + | INDEX | + +------------+ ``` \ No newline at end of file diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index ae8c9f74..43e9d985 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -185,7 +185,6 @@ public class ControlService extends Service { Spark.post("/public/actions/calculate-adjacencies", controlActionsService::calculateAdjacencies, redirectToActors); Spark.post("/public/actions/reload-blogs-list", controlActionsService::reloadBlogsList, redirectToActors); Spark.post("/public/actions/repartition-index", controlActionsService::triggerRepartition, redirectToActors); - Spark.post("/public/actions/reconstruct-index", controlActionsService::triggerIndexReconstruction, redirectToActors); Spark.post("/public/actions/trigger-data-exports", controlActionsService::triggerDataExports, redirectToActors); Spark.post("/public/actions/flush-api-caches", controlActionsService::flushApiCaches, redirectToActors); Spark.post("/public/actions/truncate-links-database", controlActionsService::truncateLinkDatabase, redirectToActors); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java index d44571a5..3d57e677 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java @@ -10,7 +10,6 @@ import nu.marginalia.actor.ActorStateFactory; import nu.marginalia.control.process.ProcessOutboxes; import nu.marginalia.control.process.ProcessService; import nu.marginalia.control.svc.BackupService; -import nu.marginalia.db.storage.model.FileStorage; import nu.marginalia.index.client.IndexClient; import nu.marginalia.index.client.IndexMqEndpoints; import nu.marginalia.mqapi.converting.ConvertAction; @@ -29,15 +28,11 @@ import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorResumeBehavior; import nu.marginalia.search.client.SearchClient; import nu.marginalia.search.client.SearchMqEndpoints; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.github.luben.zstd.ZstdOutputStream; + import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; import java.sql.SQLException; -import java.time.LocalDateTime; @Singleton public class ConvertAndLoadActor extends AbstractActorPrototype { @@ -298,7 +293,7 @@ public class ConvertAndLoadActor extends AbstractActorPrototype { public void switchOver(Long id) throws Exception { // Notify services to switch over searchOutbox.sendNotice(SearchMqEndpoints.SWITCH_LINKDB, ":-)"); - indexOutbox.sendNotice(IndexMqEndpoints.INDEX_REINDEX, ":^D"); + indexOutbox.sendNotice(IndexMqEndpoints.SWITCH_INDEX, ":^D"); } } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java index 1f191a62..a8267d40 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActionsService.java @@ -4,14 +4,12 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.control.actor.ControlActors; import nu.marginalia.control.actor.Actor; -import nu.marginalia.db.DbDomainQueries; import nu.marginalia.db.DomainTypes; import nu.marginalia.index.client.IndexClient; import nu.marginalia.index.client.IndexMqEndpoints; import nu.marginalia.mq.MessageQueueFactory; import nu.marginalia.mq.outbox.MqOutbox; import nu.marginalia.search.client.SearchClient; -import nu.marginalia.search.client.SearchMqEndpoints; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.id.ServiceId; import spark.Request; @@ -24,7 +22,6 @@ import java.util.UUID; public class ControlActionsService { private final ControlActors actors; - private final SearchClient searchClient; private final IndexClient indexClient; private final MqOutbox apiOutbox; private final ServiceEventLog eventLog; @@ -32,14 +29,12 @@ public class ControlActionsService { @Inject public ControlActionsService(ControlActors actors, - SearchClient searchClient, IndexClient indexClient, MessageQueueFactory mqFactory, ServiceEventLog eventLog, DomainTypes domainTypes) { this.actors = actors; - this.searchClient = searchClient; this.indexClient = indexClient; this.apiOutbox = createApiOutbox(mqFactory); this.eventLog = eventLog; @@ -107,10 +102,4 @@ public class ControlActionsService { return null; } - - public Object triggerIndexReconstruction(Request request, Response response) throws Exception { - indexClient.outbox().sendAsync(IndexMqEndpoints.INDEX_REINDEX, ""); - - return null; - } } diff --git a/code/services-core/control-service/src/main/resources/templates/control/actions.hdb b/code/services-core/control-service/src/main/resources/templates/control/actions.hdb index 7e3a388b..9ec528d4 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/actions.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/actions.hdb @@ -45,16 +45,6 @@ - - Reconstruct Index

- This will reconstruct the index from the index journal. - - -

- -
- - Flush api-service Caches

This will instruct the api-service to flush its caches, @@ -62,7 +52,7 @@ changes to the API licenses directly through the database. -

+
diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java index 3729c0dd..7d3b0f85 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java @@ -20,7 +20,6 @@ import spark.Request; import spark.Response; import spark.Spark; -import java.io.IOException; import java.util.concurrent.TimeUnit; import static spark.Spark.get; @@ -75,12 +74,6 @@ public class IndexService extends Service { volatile boolean initialized = false; - @MqRequest(endpoint = IndexMqEndpoints.INDEX_RELOAD_LEXICON) - public String reloadLexicon(String message) throws Exception { - throw new UnsupportedOperationException(); - } - - @MqRequest(endpoint = IndexMqEndpoints.INDEX_REPARTITION) public String repartition(String message) { if (!opsService.repartition()) { @@ -89,9 +82,9 @@ public class IndexService extends Service { return "ok"; } - @MqNotification(endpoint = IndexMqEndpoints.INDEX_REINDEX) - public String reindex(String message) throws Exception { - if (!opsService.reindex()) { + @MqNotification(endpoint = IndexMqEndpoints.SWITCH_INDEX) + public String switchIndex(String message) throws Exception { + if (!opsService.switchIndex()) { throw new IllegalStateException("Ops lock busy"); } diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java index e5e20af8..f0417e53 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexServicesFactory.java @@ -23,14 +23,11 @@ public class IndexServicesFactory { private final Logger logger = LoggerFactory.getLogger(getClass()); private final Path searchSetsBase; - final int LIVE_PART = 0; - final int NEXT_PART = 1; - @Inject public IndexServicesFactory( ServiceHeartbeat heartbeat, FileStorageService fileStorageService - ) throws IOException, SQLException { + ) throws SQLException { liveStorage = fileStorageService.getStorageByType(FileStorageType.INDEX_LIVE).asPath(); searchSetsBase = fileStorageService.getStorageByType(FileStorageType.SEARCH_SETS).asPath(); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java index e7701d79..397e124e 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexOpsService.java @@ -33,10 +33,8 @@ public class IndexOpsService { public boolean repartition() { return run(searchSetService::recalculateAll); } - public boolean reindex() throws Exception { - return run(() -> { - return index.switchIndex(); - }).isPresent(); + public boolean switchIndex() throws Exception { + return run(index::switchIndex).isPresent(); }