From 108b4cb648fbeca9db1323ae74e755c082382e58 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sat, 14 Oct 2023 20:58:55 +0200 Subject: [PATCH] (service) Keep disabled multi-noded services dormant when they are configured to be disabled. --- code/common/service/build.gradle | 1 + .../service/server/NodeStatusWatcher.java | 97 +++++++++++++++++++ .../nu/marginalia/executor/ExecutorMain.java | 2 + .../java/nu/marginalia/index/IndexMain.java | 3 + 4 files changed, 103 insertions(+) create mode 100644 code/common/service/src/main/java/nu/marginalia/service/server/NodeStatusWatcher.java diff --git a/code/common/service/build.gradle b/code/common/service/build.gradle index e26900cc..5ae203bd 100644 --- a/code/common/service/build.gradle +++ b/code/common/service/build.gradle @@ -14,6 +14,7 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:libraries:message-queue') implementation project(':code:common:db') + implementation project(':code:common:config') implementation libs.spark implementation libs.guice diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/NodeStatusWatcher.java b/code/common/service/src/main/java/nu/marginalia/service/server/NodeStatusWatcher.java new file mode 100644 index 00000000..78882eb1 --- /dev/null +++ b/code/common/service/src/main/java/nu/marginalia/service/server/NodeStatusWatcher.java @@ -0,0 +1,97 @@ +package nu.marginalia.service.server; + +import com.google.inject.name.Named; +import jakarta.inject.Inject; +import nu.marginalia.nodecfg.NodeConfigurationService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +/** The node status watcher ensures that services that can be run on multiple nodes + * find the configuration they expect, and kills the services when a node is disabled. + *

+ * Install the watcher by adding to the Main class an + *
+ * injector.getInstance(NodeStatusWatcher.class); + *
+ * before anything else is initialized. + */ +public class NodeStatusWatcher { + private static final Logger logger = LoggerFactory.getLogger(NodeStatusWatcher.class); + + private final NodeConfigurationService configurationService; + private final int nodeId; + + private final Duration pollDuration = Duration.ofSeconds(15); + + @Inject + public NodeStatusWatcher(NodeConfigurationService configurationService, + @Named("wmsa-system-node") Integer nodeId) throws InterruptedException { + this.configurationService = configurationService; + + this.nodeId = nodeId; + + awaitConfiguration(); + + + var watcherThread = new Thread(this::watcher, "node watcher"); + watcherThread.setDaemon(true); + watcherThread.start(); + } + + /** Wait for the presence of an enabled NodeConfiguration before permitting the service to start */ + private void awaitConfiguration() throws InterruptedException { + + boolean complained = false; + + for (;;) { + try { + var config = configurationService.get(nodeId); + if (null != config && !config.disabled()) { + return; + } + else if (!complained) { + logger.info("Waiting for node configuration, id = {}", nodeId); + complained = true; + } + } + catch (SQLException ex) { + logger.error("Error updating node status", ex); + } + + TimeUnit.SECONDS.sleep(pollDuration.toSeconds()); + } + + } + + /** Look for changes in the configuration and kill the service if the corresponding + * NodeConfiguration is set to be disabled. + */ + private void watcher() { + for (;;) { + try { + TimeUnit.SECONDS.sleep(pollDuration.toSeconds()); + } + catch (InterruptedException ex) { + logger.error("Watcher thread interrupted", ex); + return; + } + + try { + var config = configurationService.get(nodeId); + if (null == config || config.disabled()) { + logger.info("Current node disabled!! Shutting down!"); + System.exit(0); + } + } + catch (SQLException ex) { + logger.error("Error updating node status", ex); + } + + } + } + +} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java index dd413699..768fd3a4 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java @@ -9,6 +9,7 @@ import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.DatabaseModule; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.server.Initialization; +import nu.marginalia.service.server.NodeStatusWatcher; public class ExecutorMain extends MainClass { private final ExecutorSvc service; @@ -26,6 +27,7 @@ public class ExecutorMain extends MainClass { new DatabaseModule(), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Executor) ); + injector.getInstance(NodeStatusWatcher.class); injector.getInstance(ExecutorMain.class); injector.getInstance(Initialization.class).setReady(); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java index 0a7ef95a..e542aee6 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java @@ -9,6 +9,7 @@ import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.module.ServiceConfigurationModule; import nu.marginalia.service.module.DatabaseModule; import nu.marginalia.service.server.Initialization; +import nu.marginalia.service.server.NodeStatusWatcher; public class IndexMain extends MainClass { private final IndexService service; @@ -27,6 +28,8 @@ public class IndexMain extends MainClass { new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Index) ); + injector.getInstance(NodeStatusWatcher.class); + injector.getInstance(IndexMain.class); injector.getInstance(Initialization.class).setReady();