Merge pull request #90 from MarginaliaSearch/run-outside-docker

Run outside of Docker

build.gradle

@@ -6,7 +6,7 @@ plugins {
 
     // This is a workaround for a bug in the Jib plugin that causes it to stall randomly
     // https://github.com/GoogleContainerTools/jib/issues/3347
-    id 'com.google.cloud.tools.jib' version '3.4.1' apply(false)
+    id 'com.google.cloud.tools.jib' version '3.4.2' apply(false)
 }
 
 group 'marginalia'

code/common/config/java/nu/marginalia/WmsaHome.java

@@ -10,7 +10,6 @@ import java.util.stream.Stream;
 
 public class WmsaHome {
     public static UserAgent getUserAgent()  {
-
         return new UserAgent(
                 System.getProperty("crawler.userAgentString", "Mozilla/5.0 (compatible; Marginalia-like bot; +https://git.marginalia.nu/))"),
                 System.getProperty("crawler.userAgentIdentifier", "search.marginalia.nu")
@@ -40,7 +39,19 @@ public class WmsaHome {
                 .findFirst();
 
         if (retStr.isEmpty()) {
-            // Check if we are running in a test environment
+            // Check parent directories for a fingerprint of the project's installation boilerplate
+            var prodRoot = Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent)
+                    .filter(p -> Files.exists(p.resolve("conf/properties/system.properties")))
+                    .filter(p -> Files.exists(p.resolve("model/tfreq-new-algo3.bin")))
+                    .findAny();
+            if (prodRoot.isPresent()) {
+                return prodRoot.get();
+            }
+
+            // Check if we are running in a test environment by looking for fingerprints
+            // matching the base of the source tree for the project, then looking up the
+            // run directory which contains a template for the installation we can use as
+            // though it's the project root for testing purposes
 
             var testRoot = Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent)
                     .filter(p -> Files.exists(p.resolve("run/env")))
@@ -50,8 +61,8 @@ public class WmsaHome {
 
             return testRoot.orElseThrow(() -> new IllegalStateException("""
                             Could not find $WMSA_HOME, either set environment
-                            variable, the 'system.homePath' property,
+                            variable, the 'system.homePath' java property,
-                            or ensure either /wmssa or /var/lib/wmsa exists
+                            or ensure either /wmsa or /var/lib/wmsa exists
                             """));
         }
 

code/common/service/java/nu/marginalia/service/discovery/ZkServiceRegistry.java

@@ -140,13 +140,15 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
     @Override
     public int requestPort(String externalHost,
                            ServiceKey<?> key) {
+
         if (!Boolean.getBoolean("service.random-port")) {
             return switch (key) {
-                case ServiceKey.Rest rest -> 80;
+                case ServiceKey.Rest rest -> Integer.getInteger("service.http-port", 80);
-                case ServiceKey.Grpc<?> grpc -> 81;
+                case ServiceKey.Grpc<?> grpc -> Integer.getInteger("service.grpc-port",81);
             };
         }
 
+
         int portRangeLow = 12_000;
         int portRangeHigh = 12_999;
 

code/common/service/java/nu/marginalia/service/module/ServiceConfigurationModule.java

@@ -36,11 +36,15 @@ public class ServiceConfigurationModule extends AbstractModule {
 
     private int getPrometheusPort() {
         String prometheusPortEnv = System.getenv("WMSA_PROMETHEUS_PORT");
-
         if (prometheusPortEnv != null) {
             return Integer.parseInt(prometheusPortEnv);
         }
 
+        Integer prometheusPortProperty = Integer.getInteger("service.prometheus-port");
+        if (prometheusPortProperty != null) {
+            return prometheusPortProperty;
+        }
+
         return 7000;
     }
 

code/common/service/java/nu/marginalia/service/server/MetricsServer.java

@@ -13,6 +13,10 @@ public class MetricsServer {
     @SneakyThrows
     @Inject
     public MetricsServer(ServiceConfiguration configuration) {
+        // If less than zero, we forego setting up a metrics server
+        if (configuration.metricsPort() < 0)
+            return;
+
         Server server = new Server(configuration.metricsPort());
         ServletContextHandler context = new ServletContextHandler();
         context.setContextPath("/");

code/common/service/resources/log4j2-json.xml

@@ -7,7 +7,7 @@
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
-        <RollingFile name="LogToFile" fileName="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
+        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
                      ignoreExceptions="false">
             <JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true" includeTimeMillis="true"/>
             <Filters>

code/common/service/resources/log4j2-prod.xml

@@ -7,7 +7,7 @@
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
-        <RollingFile name="LogToFile" fileName="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
+        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
                      ignoreExceptions="false">
             <PatternLayout>
                 <Pattern>%-5level %d{yyyy-MM-dd HH:mm:ss,SSS} %-20t %-20c{1}: %msg{nolookups}%n</Pattern>

code/common/service/resources/log4j2-test.xml

@@ -6,7 +6,7 @@
                 <MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
             </Filters>
         </Console>
-        <RollingFile name="LogToFile" fileName="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
+        <RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
                      ignoreExceptions="false">
             <PatternLayout>
                 <Pattern>%-5level %d{yyyy-MM-dd HH:mm:ss,SSS} %-20t %-20c{1}: %msg{nolookups}%n</Pattern>

code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java

@@ -94,6 +94,7 @@ public class IndexResultValuationContext {
 
         CompiledQueryLong wordMetasQuery = new CompiledQueryLong(compiledQuery.root, new CqDataLong(wordMetas));
 
+
         boolean allSynthetic = CompiledQueryAggregates.booleanAggregate(wordMetasQuery, WordFlags.Synthetic::isPresent);
         int flagsCount = CompiledQueryAggregates.intMaxMinAggregate(wordMetasQuery, wordMeta -> Long.bitCount(wordMeta & flagsFilterMask));
         int positionsCount = CompiledQueryAggregates.intMaxMinAggregate(wordMetasQuery, wordMeta -> Long.bitCount(WordMetadata.decodePositions(wordMeta)));

code/services-application/api-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 java {

code/services-application/api-service/java/nu/marginalia/api/ApiMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -23,6 +24,10 @@ public class ApiMain extends MainClass {
                 new DatabaseModule(false),
                 new ServiceDiscoveryModule(),
                 new ServiceConfigurationModule(ServiceId.Api));
+
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(ApiMain.class);
         injector.getInstance(Initialization.class).setReady();
     }

code/services-application/dating-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 application {

code/services-application/dating-service/java/nu/marginalia/dating/DatingMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -31,6 +32,9 @@ public class DatingMain extends MainClass {
                 new DatabaseModule(false)
         );
 
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(DatingMain.class);
         injector.getInstance(Initialization.class).setReady();
     }

code/services-application/explorer-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 application {

code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -31,6 +32,9 @@ public class ExplorerMain extends MainClass {
                 new DatabaseModule(false)
         );
 
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(ExplorerMain.class);
         injector.getInstance(Initialization.class).setReady();
     }

code/services-application/search-service/build.gradle

@@ -5,7 +5,7 @@ plugins {
     id 'application'
     id 'jvm-test-suite'
 
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 application {

code/services-application/search-service/java/nu/marginalia/search/SearchMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -32,6 +33,9 @@ public class SearchMain extends MainClass {
                 new DatabaseModule(false)
         );
 
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(SearchMain.class);
         injector.getInstance(Initialization.class).setReady();
 

code/services-core/assistant-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 application {

code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -28,6 +29,10 @@ public class AssistantMain extends MainClass {
                 new DatabaseModule(false)
         );
 
+
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(AssistantMain.class);
         injector.getInstance(Initialization.class).setReady();
 

code/services-core/control-service/build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'java'
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 java {

code/services-core/control-service/java/nu/marginalia/control/ControlMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -25,6 +26,9 @@ public class ControlMain extends MainClass {
                 new ServiceDiscoveryModule(),
                 new ServiceConfigurationModule(ServiceId.Control));
 
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(ControlMain.class);
         injector.getInstance(Initialization.class).setReady();
     }

code/services-core/executor-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 application {
@@ -13,8 +13,6 @@ application {
 
 tasks.distZip.enabled = false
 
-
-
 java {
     toolchain {
         languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))

code/services-core/executor-service/java/nu/marginalia/executor/ExecutorMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.DatabaseModule;
@@ -28,8 +29,11 @@ public class ExecutorMain extends MainClass  {
                 new ServiceDiscoveryModule(),
                 new ServiceConfigurationModule(ServiceId.Executor)
         );
-        injector.getInstance(NodeStatusWatcher.class);
 
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
+        injector.getInstance(NodeStatusWatcher.class);
         injector.getInstance(ExecutorMain.class);
         injector.getInstance(Initialization.class).setReady();
     }

code/services-core/index-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 application {

code/services-core/index-service/java/nu/marginalia/index/IndexMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -29,6 +30,9 @@ public class IndexMain extends MainClass {
                 new ServiceConfigurationModule(ServiceId.Index)
         );
 
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(NodeStatusWatcher.class);
 
         injector.getInstance(IndexMain.class);

code/services-core/query-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 application {

code/services-core/query-service/java/nu/marginalia/query/QueryMain.java

@@ -4,6 +4,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.module.ServiceDiscoveryModule;
 import nu.marginalia.service.ServiceId;
 import nu.marginalia.service.module.ServiceConfigurationModule;
@@ -28,6 +29,9 @@ public class QueryMain extends MainClass {
                 new ServiceConfigurationModule(ServiceId.Query)
         );
 
+        // Ensure that the service registry is initialized early
+        injector.getInstance(ServiceRegistryIf.class);
+
         injector.getInstance(QueryMain.class);
         injector.getInstance(Initialization.class).setReady();
     }

code/services-core/single-service-runner/build.gradle

@@ -0,0 +1,34 @@
+plugins {
+    id 'java'
+    id 'application'
+    id 'jvm-test-suite'
+}
+
+application {
+    mainClass = 'nu.marginalia.SingleService'
+    applicationName = 'marginalia'
+    applicationDefaultJvmArgs = [ "--enable-preview" ]
+}
+
+tasks.distZip.enabled = false
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))
+    }
+}
+
+apply from: "$rootProject.projectDir/srcsets.gradle"
+
+dependencies {
+    implementation project(':code:services-core:query-service')
+    implementation project(':code:services-core:index-service')
+    implementation project(':code:services-core:control-service')
+    implementation project(':code:services-core:executor-service')
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+}
+

code/services-core/single-service-runner/java/nu/marginalia/SingleService.java

@@ -0,0 +1,112 @@
+package nu.marginalia;
+
+/** Springboard for launching services outside of docker */
+public class SingleService {
+
+    public static void main(String... args) {
+        if (!configure(args)) {
+            System.out.println("Usage: SingleService <service> bind-address:bind-port-http:bind-port-grpc announce-address [args...]");
+            return;
+        }
+
+        requireEnv("ZOOKEEPER_HOSTS", "Comma-separated list of zookeeper hosts");
+        requireEnv("WMSA_HOME", "Path to the install directory of the project");
+
+        String serviceName = args[0];
+        String[] serviceArgs = new String[args.length - 3];
+        System.arraycopy(args, 3, serviceArgs, 0, serviceArgs.length);
+
+        for (var service : Service.values()) {
+            if (service.name.equals(serviceName)) {
+                service.run(serviceArgs);
+            }
+        }
+    }
+
+    private static void requireEnv(String env, String desc) {
+        if (System.getenv(env) == null) {
+            throw new IllegalArgumentException("Missing environment variable: " + env + " - " + desc);
+        }
+        else {
+            System.out.println("Found environment variable: " + env + " = " + System.getenv(env));
+        }
+    }
+
+    /** Set system properties for the address and ports for the service.
+     *
+     * @return true if the configuration was successful
+     * */
+    private static boolean configure(String[] args) {
+        if (args.length < 3)
+            return false;
+
+        try {
+            final String bindAddress_http_grpc = args[1];
+            final String announceAddress = args[2];
+
+            final String[] bindParts = bindAddress_http_grpc.split(":");
+            if (bindParts.length < 3)
+                return false;
+
+            String bindAddress = bindParts[0];
+
+            int httpPort = Integer.parseInt(bindParts[1]);
+            int grpcPort = Integer.parseInt(bindParts[2]);
+
+            System.out.println(STR."""
+                Configuring service with bind address: \{bindAddress}
+                                         http port: \{httpPort}
+                                         grpc port: \{grpcPort}
+                                         announce address: \{announceAddress}
+                """);
+
+            System.setProperty("service.bind-address", bindAddress);
+            System.setProperty("service.http-port", Integer.toString(httpPort));
+            System.setProperty("service.grpc-port", Integer.toString(grpcPort));
+            System.setProperty("service.host", announceAddress);
+
+            // By default, disable prometheus metrics
+            if (System.getProperty("service.prometheus-port") == null) {
+                System.setProperty("service.prometheus-port", "-1");
+            }
+
+            return true;
+        }
+        catch (NumberFormatException e) {
+            return false;
+        }
+
+    }
+
+    enum Service {
+        IndexService("index", "nu.marginalia.index.IndexMain"),
+        ControlService("control", "nu.marginalia.control.ControlMain"),
+        ExecutorService("executor", "nu.marginalia.executor.ExecutorMain"),
+        QueryService("query", "nu.marginalia.query.QueryMain"),
+        ;
+
+        public final String name;
+        public final String className;
+
+        Service(String name, String className) {
+            this.name = name;
+            this.className = className;
+        }
+
+        /** Call the main method of the service class */
+        public void run(String[] args) {
+            try {
+                // Use reflection to call the main method of the service class to avoid
+                // loading all classes at startup time, which would invoke a bunch of contradictory
+                // static initializers
+
+                Class<?> clazz = Class.forName(className);
+                clazz.getMethod("main", String[].class).invoke(null, (Object) args);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+
+}

code/tools/screenshot-capture-tool/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.google.cloud.tools.jib' version '3.4.1'
+    id 'com.google.cloud.tools.jib' version '3.4.2'
 }
 
 java {

settings.gradle

@@ -5,6 +5,7 @@ include 'code:services-core:assistant-service'
 include 'code:services-core:control-service'
 include 'code:services-core:query-service'
 include 'code:services-core:executor-service'
+include 'code:services-core:single-service-runner'
 
 include 'code:services-application:search-service'
 include 'code:services-application:api-service'