Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #90 from MarginaliaSearch/run-outside-docker

Browse Source 
Run outside of Docker
This commit is contained in:
Viktor 2024-04-25 18:55:26 +02:00 committed by GitHub
commit 0f41105436
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
31 changed files with 229 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -6,7 +6,7 @@ plugins {
// This is a workaround for a bug in the Jib plugin that causes it to stall randomly
// https://github.com/GoogleContainerTools/jib/issues/3347
id 'com.google.cloud.tools.jib' version '3.4.1' apply(false)
id 'com.google.cloud.tools.jib' version '3.4.2' apply(false)
}
group 'marginalia'

19
code/common/config/java/nu/marginalia/WmsaHome.java
View File

@ -10,7 +10,6 @@ import java.util.stream.Stream;
public class WmsaHome {
public static UserAgent getUserAgent() {
return new UserAgent(
System.getProperty("crawler.userAgentString", "Mozilla/5.0 (compatible; Marginalia-like bot; +https://git.marginalia.nu/))"),
System.getProperty("crawler.userAgentIdentifier", "search.marginalia.nu")
@ -40,7 +39,19 @@ public class WmsaHome {
.findFirst();
if (retStr.isEmpty()) {
// Check if we are running in a test environment
// Check parent directories for a fingerprint of the project's installation boilerplate
var prodRoot = Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent)
.filter(p -> Files.exists(p.resolve("conf/properties/system.properties")))
.filter(p -> Files.exists(p.resolve("model/tfreq-new-algo3.bin")))
.findAny();
if (prodRoot.isPresent()) {
return prodRoot.get();
}
// Check if we are running in a test environment by looking for fingerprints
// matching the base of the source tree for the project, then looking up the
// run directory which contains a template for the installation we can use as
// though it's the project root for testing purposes
var testRoot = Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent)
.filter(p -> Files.exists(p.resolve("run/env")))
@ -50,8 +61,8 @@ public class WmsaHome {
return testRoot.orElseThrow(() -> new IllegalStateException("""
Could not find $WMSA_HOME, either set environment
variable, the 'system.homePath' property,
or ensure either /wmssa or /var/lib/wmsa exists
variable, the 'system.homePath' java property,
or ensure either /wmsa or /var/lib/wmsa exists
"""));
}

6
code/common/service/java/nu/marginalia/service/discovery/ZkServiceRegistry.java
View File

@ -140,13 +140,15 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
@Override
public int requestPort(String externalHost,
ServiceKey<?> key) {
if (!Boolean.getBoolean("service.random-port")) {
return switch (key) {
case ServiceKey.Rest rest -> 80;
case ServiceKey.Grpc<?> grpc -> 81;
case ServiceKey.Rest rest -> Integer.getInteger("service.http-port", 80);
case ServiceKey.Grpc<?> grpc -> Integer.getInteger("service.grpc-port",81);
};
}
int portRangeLow = 12_000;
int portRangeHigh = 12_999;

6
code/common/service/java/nu/marginalia/service/module/ServiceConfigurationModule.java
View File

@ -36,11 +36,15 @@ public class ServiceConfigurationModule extends AbstractModule {
private int getPrometheusPort() {
String prometheusPortEnv = System.getenv("WMSA_PROMETHEUS_PORT");
if (prometheusPortEnv != null) {
return Integer.parseInt(prometheusPortEnv);
}
Integer prometheusPortProperty = Integer.getInteger("service.prometheus-port");
if (prometheusPortProperty != null) {
return prometheusPortProperty;
}
return 7000;
}

4
code/common/service/java/nu/marginalia/service/server/MetricsServer.java
View File

@ -13,6 +13,10 @@ public class MetricsServer {
@SneakyThrows
@Inject
public MetricsServer(ServiceConfiguration configuration) {
// If less than zero, we forego setting up a metrics server
if (configuration.metricsPort() < 0)
return;
Server server = new Server(configuration.metricsPort());
ServletContextHandler context = new ServletContextHandler();
context.setContextPath("/");

2
code/common/service/resources/log4j2-json.xml
View File

@ -7,7 +7,7 @@
<MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
</Filters>
</Console>
<RollingFile name="LogToFile" fileName="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
<RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
ignoreExceptions="false">
<JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true" includeTimeMillis="true"/>
<Filters>

2
code/common/service/resources/log4j2-prod.xml
View File

@ -7,7 +7,7 @@
<MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
</Filters>
</Console>
<RollingFile name="LogToFile" fileName="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
<RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
ignoreExceptions="false">
<PatternLayout>
<Pattern>%-5level %d{yyyy-MM-dd HH:mm:ss,SSS} %-20t %-20c{1}: %msg{nolookups}%n</Pattern>

2
code/common/service/resources/log4j2-test.xml
View File

@ -6,7 +6,7 @@
<MarkerFilter marker="HTTP" onMatch="DENY" onMismatch="NEUTRAL" />
</Filters>
</Console>
<RollingFile name="LogToFile" fileName="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
<RollingFile name="LogToFile" fileName="${env:WMSA_LOG_DIR:/var/log/wmsa}/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}.log" filePattern="/var/log/wmsa/wmsa-${sys:service-name}-${env:WMSA_SERVICE_NODE:-0}-log-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz"
ignoreExceptions="false">
<PatternLayout>
<Pattern>%-5level %d{yyyy-MM-dd HH:mm:ss,SSS} %-20t %-20c{1}: %msg{nolookups}%n</Pattern>

1
code/index/java/nu/marginalia/index/results/IndexResultValuationContext.java
View File

@ -94,6 +94,7 @@ public class IndexResultValuationContext {
CompiledQueryLong wordMetasQuery = new CompiledQueryLong(compiledQuery.root, new CqDataLong(wordMetas));
boolean allSynthetic = CompiledQueryAggregates.booleanAggregate(wordMetasQuery, WordFlags.Synthetic::isPresent);
int flagsCount = CompiledQueryAggregates.intMaxMinAggregate(wordMetasQuery, wordMeta -> Long.bitCount(wordMeta & flagsFilterMask));
int positionsCount = CompiledQueryAggregates.intMaxMinAggregate(wordMetasQuery, wordMeta -> Long.bitCount(WordMetadata.decodePositions(wordMeta)));

2
code/services-application/api-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
java {

5
code/services-application/api-service/java/nu/marginalia/api/ApiMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -23,6 +24,10 @@ public class ApiMain extends MainClass {
new DatabaseModule(false),
new ServiceDiscoveryModule(),
new ServiceConfigurationModule(ServiceId.Api));
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(ApiMain.class);
injector.getInstance(Initialization.class).setReady();
}

2
code/services-application/dating-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
application {

4
code/services-application/dating-service/java/nu/marginalia/dating/DatingMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -31,6 +32,9 @@ public class DatingMain extends MainClass {
new DatabaseModule(false)
);
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(DatingMain.class);
injector.getInstance(Initialization.class).setReady();
}

2
code/services-application/explorer-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
application {

4
code/services-application/explorer-service/java/nu/marginalia/explorer/ExplorerMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -31,6 +32,9 @@ public class ExplorerMain extends MainClass {
new DatabaseModule(false)
);
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(ExplorerMain.class);
injector.getInstance(Initialization.class).setReady();
}

2
code/services-application/search-service/build.gradle
View File

@ -5,7 +5,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
application {

4
code/services-application/search-service/java/nu/marginalia/search/SearchMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -32,6 +33,9 @@ public class SearchMain extends MainClass {
new DatabaseModule(false)
);
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(SearchMain.class);
injector.getInstance(Initialization.class).setReady();

2
code/services-core/assistant-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
application {

5
code/services-core/assistant-service/java/nu/marginalia/assistant/AssistantMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -28,6 +29,10 @@ public class AssistantMain extends MainClass {
new DatabaseModule(false)
);
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(AssistantMain.class);
injector.getInstance(Initialization.class).setReady();

2
code/services-core/control-service/build.gradle
View File

@ -2,7 +2,7 @@ plugins {
id 'java'
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
java {

4
code/services-core/control-service/java/nu/marginalia/control/ControlMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -25,6 +26,9 @@ public class ControlMain extends MainClass {
new ServiceDiscoveryModule(),
new ServiceConfigurationModule(ServiceId.Control));
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(ControlMain.class);
injector.getInstance(Initialization.class).setReady();
}

4
code/services-core/executor-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
application {
@ -13,8 +13,6 @@ application {
tasks.distZip.enabled = false
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))

6
code/services-core/executor-service/java/nu/marginalia/executor/ExecutorMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.DatabaseModule;
@ -28,8 +29,11 @@ public class ExecutorMain extends MainClass {
new ServiceDiscoveryModule(),
new ServiceConfigurationModule(ServiceId.Executor)
);
injector.getInstance(NodeStatusWatcher.class);
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(NodeStatusWatcher.class);
injector.getInstance(ExecutorMain.class);
injector.getInstance(Initialization.class).setReady();
}

2
code/services-core/index-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
application {

4
code/services-core/index-service/java/nu/marginalia/index/IndexMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -29,6 +30,9 @@ public class IndexMain extends MainClass {
new ServiceConfigurationModule(ServiceId.Index)
);
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(NodeStatusWatcher.class);
injector.getInstance(IndexMain.class);

2
code/services-core/query-service/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
application {

4
code/services-core/query-service/java/nu/marginalia/query/QueryMain.java
View File

@ -4,6 +4,7 @@ import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.service.MainClass;
import nu.marginalia.service.discovery.ServiceRegistryIf;
import nu.marginalia.service.module.ServiceDiscoveryModule;
import nu.marginalia.service.ServiceId;
import nu.marginalia.service.module.ServiceConfigurationModule;
@ -28,6 +29,9 @@ public class QueryMain extends MainClass {
new ServiceConfigurationModule(ServiceId.Query)
);
// Ensure that the service registry is initialized early
injector.getInstance(ServiceRegistryIf.class);
injector.getInstance(QueryMain.class);
injector.getInstance(Initialization.class).setReady();
}

34
code/services-core/single-service-runner/build.gradle Normal file
View File

@ -0,0 +1,34 @@
plugins {
id 'java'
id 'application'
id 'jvm-test-suite'
}
application {
mainClass = 'nu.marginalia.SingleService'
applicationName = 'marginalia'
applicationDefaultJvmArgs = [ "--enable-preview" ]
}
tasks.distZip.enabled = false
java {
toolchain {
languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))
}
}
apply from: "$rootProject.projectDir/srcsets.gradle"
dependencies {
implementation project(':code:services-core:query-service')
implementation project(':code:services-core:index-service')
implementation project(':code:services-core:control-service')
implementation project(':code:services-core:executor-service')
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito
}

112
code/services-core/single-service-runner/java/nu/marginalia/SingleService.java Normal file
View File

@ -0,0 +1,112 @@
package nu.marginalia;
/** Springboard for launching services outside of docker */
public class SingleService {
public static void main(String... args) {
if (!configure(args)) {
System.out.println("Usage: SingleService <service> bind-address:bind-port-http:bind-port-grpc announce-address [args...]");
return;
}
requireEnv("ZOOKEEPER_HOSTS", "Comma-separated list of zookeeper hosts");
requireEnv("WMSA_HOME", "Path to the install directory of the project");
String serviceName = args[0];
String[] serviceArgs = new String[args.length - 3];
System.arraycopy(args, 3, serviceArgs, 0, serviceArgs.length);
for (var service : Service.values()) {
if (service.name.equals(serviceName)) {
service.run(serviceArgs);
}
}
}
private static void requireEnv(String env, String desc) {
if (System.getenv(env) == null) {
throw new IllegalArgumentException("Missing environment variable: " + env + " - " + desc);
}
else {
System.out.println("Found environment variable: " + env + " = " + System.getenv(env));
}
}
/** Set system properties for the address and ports for the service.
*
* @return true if the configuration was successful
* */
private static boolean configure(String[] args) {
if (args.length < 3)
return false;
try {
final String bindAddress_http_grpc = args[1];
final String announceAddress = args[2];
final String[] bindParts = bindAddress_http_grpc.split(":");
if (bindParts.length < 3)
return false;
String bindAddress = bindParts[0];
int httpPort = Integer.parseInt(bindParts[1]);
int grpcPort = Integer.parseInt(bindParts[2]);
System.out.println(STR."""
Configuring service with bind address: \{bindAddress}
http port: \{httpPort}
grpc port: \{grpcPort}
announce address: \{announceAddress}
""");
System.setProperty("service.bind-address", bindAddress);
System.setProperty("service.http-port", Integer.toString(httpPort));
System.setProperty("service.grpc-port", Integer.toString(grpcPort));
System.setProperty("service.host", announceAddress);
// By default, disable prometheus metrics
if (System.getProperty("service.prometheus-port") == null) {
System.setProperty("service.prometheus-port", "-1");
}
return true;
}
catch (NumberFormatException e) {
return false;
}
}
enum Service {
IndexService("index", "nu.marginalia.index.IndexMain"),
ControlService("control", "nu.marginalia.control.ControlMain"),
ExecutorService("executor", "nu.marginalia.executor.ExecutorMain"),
QueryService("query", "nu.marginalia.query.QueryMain"),
;
public final String name;
public final String className;
Service(String name, String className) {
this.name = name;
this.className = className;
}
/** Call the main method of the service class */
public void run(String[] args) {
try {
// Use reflection to call the main method of the service class to avoid
// loading all classes at startup time, which would invoke a bunch of contradictory
// static initializers
Class<?> clazz = Class.forName(className);
clazz.getMethod("main", String[].class).invoke(null, (Object) args);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}

2
code/tools/screenshot-capture-tool/build.gradle
View File

@ -3,7 +3,7 @@ plugins {
id 'application'
id 'jvm-test-suite'
id 'com.google.cloud.tools.jib' version '3.4.1'
id 'com.google.cloud.tools.jib' version '3.4.2'
}
java {

1
settings.gradle
View File

@ -5,6 +5,7 @@ include 'code:services-core:assistant-service'
include 'code:services-core:control-service'
include 'code:services-core:query-service'
include 'code:services-core:executor-service'
include 'code:services-core:single-service-runner'
include 'code:services-application:search-service'
include 'code:services-application:api-service'