From 0e65384781d1d82bd2eda8e9ac096dcd5c6d319f Mon Sep 17 00:00:00 2001
From: vlofgren <vlofgren@gmail.com>
Date: Fri, 3 Jun 2022 13:32:05 +0200
Subject: [PATCH] Make WMSA_HOME configurable through an environment variable.

---
 .../nu/marginalia/wmsa/edge/E2ETestBase.java  |  2 +-
 marginalia_nu/src/e2e/resources/init.sh       | 33 ++++++++-----
 .../wmsa/configuration/WmsaHome.java          | 49 ++++++++++++++-----
 .../edge/assistant/EdgeAssistantModule.java   | 13 ++---
 .../wmsa/edge/converting/ConverterModule.java | 10 +---
 .../wmsa/edge/index/EdgeTablesModule.java     |  9 ++--
 .../wmsa/edge/search/EdgeSearchModule.java    | 13 +----
 .../resource_store/ResourceStoreModule.java   |  1 -
 8 files changed, 70 insertions(+), 60 deletions(-)

diff --git a/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java b/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java
index 769eca40..0c329a79 100644
--- a/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java
+++ b/marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java
@@ -33,7 +33,7 @@ public abstract class E2ETestBase {
                 .withCopyFileToContainer(jarFile(), "/WMSA.jar")
                 .withCopyFileToContainer(MountableFile.forClasspathResource("init.sh"), "/init.sh")
                 .withExposedPorts(service.port)
-                .withFileSystemBind(modelsPath(), "/var/lib/wmsa/model", BindMode.READ_ONLY)
+                .withFileSystemBind(modelsPath(), "/wmsa/model", BindMode.READ_ONLY)
                 .withNetwork(network)
                 .withNetworkAliases(service.name)
                 .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(service.name)))
diff --git a/marginalia_nu/src/e2e/resources/init.sh b/marginalia_nu/src/e2e/resources/init.sh
index a6aaea3b..5409f787 100644
--- a/marginalia_nu/src/e2e/resources/init.sh
+++ b/marginalia_nu/src/e2e/resources/init.sh
@@ -1,13 +1,15 @@
 #!/bin/bash
 
-mkdir -p /var/lib/wmsa/encyclopedia
-mkdir -p /var/lib/wmsa/conf
-mkdir -p /var/lib/wmsa/index/write
-mkdir -p /var/lib/wmsa/index/read
-mkdir -p /backup/work/index-tmp
+HOME=/wmsa
 
-mkdir -p /var/log/wmsa
-cat > /var/lib/wmsa/suggestions.txt <<EOF
+mkdir -p ${HOME}/encyclopedia
+mkdir -p ${HOME}/conf
+mkdir -p ${HOME}/index/write
+mkdir -p ${HOME}/index/read
+mkdir -p ${HOME}/tmp-slow
+mkdir -p ${HOME}/tmp-fast
+
+cat > ${HOME}/suggestions.txt <<EOF
 state
 three
 while
@@ -22,17 +24,22 @@ many
 year
 EOF
 
-cat > /var/lib/wmsa/conf/disks.properties <<EOF
-encyclopedia=/var/lib/wmsa/encyclopedia
+cat > ${HOME}/conf/disks.properties <<EOF
+encyclopedia=${HOME}/encyclopedia
+
+index-write=${HOME}/index/write
+index-read=${HOME}/index/read
+tmp-slow=${HOME}/tmp-slow
+tmp-fast=${HOME}/tmp-fast
 EOF
 
-cat > /var/lib/wmsa/conf/db.properties <<EOF
+cat > ${HOME}/conf/db.properties <<EOF
   db.user=wmsa
   db.pass=wmsa
   db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true
 EOF
 
-cat > /var/lib/wmsa/conf/ranking-settings.yaml <<EOF
+cat > ${HOME}/conf/ranking-settings.yaml <<EOF
 ---
 retro:
   - "%"
@@ -46,7 +53,7 @@ standard:
   - "%"
 EOF
 
-cat > /var/lib/wmsa/conf/hosts <<EOF
+cat > ${HOME}/conf/hosts <<EOF
 # service-name host-name
 resource-store resource-store
 renderer renderer
@@ -62,4 +69,4 @@ memex memex
 dating dating
 EOF
 
-java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1
\ No newline at end of file
+WMSA_HOME=${HOME} java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1
\ No newline at end of file
diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java
index 6c5bc1a1..87768ca0 100644
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java
@@ -1,9 +1,12 @@
 package nu.marginalia.wmsa.configuration;
 
+import nu.marginalia.util.language.conf.LanguageModels;
+
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Optional;
 import java.util.Properties;
 
 public class WmsaHome {
@@ -20,7 +23,9 @@ public class WmsaHome {
     }
 
     public static Path getHomePath() {
-        var ret = Path.of(System.getProperty("WMSA_HOME", DEFAULT));
+        var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse(DEFAULT);
+
+        var ret = Path.of(retStr);
         if (!Files.isDirectory(ret)) {
             throw new IllegalStateException("Could not find WMSA_HOME, either set environment variable or ensure " + DEFAULT + " exists");
         }
@@ -45,26 +50,44 @@ public class WmsaHome {
         return getHomePath().resolve("data").resolve("IP2LOCATION-LITE-DB1.CSV");
     }
 
-    public static Path getDisk(String name) throws IOException {
-        Path p = Path.of(getDiskProperties().getProperty(name));
+    public static Path getDisk(String name) {
+        var pathStr = getDiskProperties().getProperty(name);
+        if (null == pathStr) {
+            throw new RuntimeException("Disk " + name + " was not configured");
+        }
+        Path p = Path.of(pathStr);
         if (!Files.isDirectory(p)) {
-            throw new IOException(name + " does not exist!");
+            throw new RuntimeException("Disk " + name + " does not exist or is not a directory!");
         }
         return p;
     }
 
-    public static Properties getDiskProperties() throws IOException {
+    public static Properties getDiskProperties() {
         Path settingsFile = getHomePath().resolve("conf/disks.properties");
 
-        if (Files.isRegularFile(settingsFile)) {
-            try (var is = Files.newInputStream(settingsFile)) {
-                var props = new Properties();
-                props.load(is);
-                return props;
-            }
+        if (!Files.isRegularFile(settingsFile)) {
+            throw new RuntimeException("Could not find disk settings " + settingsFile);
         }
-        else {
-            throw new IOException("Could not find disk settings " + settingsFile);
+
+        try (var is = Files.newInputStream(settingsFile)) {
+            var props = new Properties();
+            props.load(is);
+            return props;
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
         }
     }
+
+    public static LanguageModels getLanguageModels() {
+        final Path home = getHomePath();
+
+        return new LanguageModels(
+                home.resolve("model/ngrams-generous-emstr.bin"),
+                home.resolve("model/tfreq-new-algo3.bin"),
+                home.resolve("model/opennlp-sentence.bin"),
+                home.resolve("model/English.RDR"),
+                home.resolve("model/English.DICT"),
+                home.resolve("model/opennlp-tok.bin"));
+    }
 }
diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/EdgeAssistantModule.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/EdgeAssistantModule.java
index cc5c3fe6..dcc8d90d 100644
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/EdgeAssistantModule.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/EdgeAssistantModule.java
@@ -2,6 +2,7 @@ package nu.marginalia.wmsa.edge.assistant;
 
 import com.google.inject.AbstractModule;
 import nu.marginalia.util.language.conf.LanguageModels;
+import nu.marginalia.wmsa.configuration.WmsaHome;
 
 import java.nio.file.Path;
 
@@ -9,14 +10,8 @@ import static com.google.inject.name.Names.named;
 
 public class EdgeAssistantModule extends AbstractModule {
     public void configure() {
-        bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(Path.of("/var/lib/wmsa/suggestions.txt"));
-        bind(LanguageModels.class).toInstance(new LanguageModels(
-                Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
-                Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
-                Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
-                Path.of("/var/lib/wmsa/model/English.RDR"),
-                Path.of("/var/lib/wmsa/model/English.DICT"),
-                Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
-        ));
+        bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(WmsaHome.getHomePath().resolve("suggestions.txt"));
+
+        bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
     }
 }
diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ConverterModule.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ConverterModule.java
index 6f03632f..4bf6eaea 100644
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ConverterModule.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ConverterModule.java
@@ -5,6 +5,7 @@ import com.google.inject.AbstractModule;
 import com.google.inject.name.Names;
 import marcono1234.gson.recordadapter.RecordTypeAdapterFactory;
 import nu.marginalia.util.language.conf.LanguageModels;
+import nu.marginalia.wmsa.configuration.WmsaHome;
 import nu.marginalia.wmsa.edge.model.EdgeCrawlPlan;
 import nu.marginalia.wmsa.edge.model.EdgeDomain;
 import nu.marginalia.wmsa.edge.model.EdgeUrl;
@@ -30,14 +31,7 @@ public class ConverterModule extends AbstractModule {
         bind(Integer.class).annotatedWith(Names.named("max-title-length")).toInstance(128);
         bind(Integer.class).annotatedWith(Names.named("max-summary-length")).toInstance(255);
 
-        bind(LanguageModels.class).toInstance(new LanguageModels(
-                Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
-                Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
-                Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
-                Path.of("/var/lib/wmsa/model/English.RDR"),
-                Path.of("/var/lib/wmsa/model/English.DICT"),
-                Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
-        ));
+        bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
     }
 
     private Gson createGson() {
diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeTablesModule.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeTablesModule.java
index bc9c2f44..4650b15b 100644
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeTablesModule.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeTablesModule.java
@@ -2,17 +2,18 @@ package nu.marginalia.wmsa.edge.index;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.name.Names;
+import nu.marginalia.wmsa.configuration.WmsaHome;
 
 import java.nio.file.Path;
 
 public class EdgeTablesModule extends AbstractModule {
 
     public void configure() {
-        bind(Path.class).annotatedWith(Names.named("partition-root-slow")).toInstance(Path.of("/var/lib/wmsa/index/write"));
-        bind(Path.class).annotatedWith(Names.named("partition-root-slow-tmp")).toInstance(Path.of("/backup/work/index-tmp/"));
+        bind(Path.class).annotatedWith(Names.named("partition-root-slow")).toInstance(WmsaHome.getDisk("index-write"));
+        bind(Path.class).annotatedWith(Names.named("partition-root-fast")).toInstance(WmsaHome.getDisk("index-read"));
 
-        bind(Path.class).annotatedWith(Names.named("partition-root-fast")).toInstance(Path.of("/var/lib/wmsa/index/read"));
-        bind(Path.class).annotatedWith(Names.named("tmp-file-dir")).toInstance(Path.of("/var/lib/wmsa/index/read"));
+        bind(Path.class).annotatedWith(Names.named("partition-root-slow-tmp")).toInstance(WmsaHome.getDisk("tmp-slow"));
+        bind(Path.class).annotatedWith(Names.named("tmp-file-dir")).toInstance(WmsaHome.getDisk("tmp-fast"));
 
         bind(String.class).annotatedWith(Names.named("edge-writer-page-index-file")).toInstance("page-index.dat");
         bind(String.class).annotatedWith(Names.named("edge-writer-dictionary-file")).toInstance("dictionary.dat");
diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchModule.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchModule.java
index 9e1df8d5..b7290d70 100644
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchModule.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchModule.java
@@ -2,21 +2,12 @@ package nu.marginalia.wmsa.edge.search;
 
 import com.google.inject.AbstractModule;
 import nu.marginalia.util.language.conf.LanguageModels;
-
-import java.nio.file.Path;
+import nu.marginalia.wmsa.configuration.WmsaHome;
 
 public class EdgeSearchModule extends AbstractModule {
 
     public void configure() {
-
-        bind(LanguageModels.class).toInstance(new LanguageModels(
-                Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
-                Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
-                Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
-                Path.of("/var/lib/wmsa/model/English.RDR"),
-                Path.of("/var/lib/wmsa/model/English.DICT"),
-                Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
-        ));
+        bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
     }
 
 }
diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java
index 2de9e931..30bac9d3 100644
--- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java
+++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java
@@ -7,7 +7,6 @@ import java.nio.file.Path;
 
 public class ResourceStoreModule extends AbstractModule {
     public void configure() {
-        bind(String.class).annotatedWith(Names.named("external-url")).toInstance("https://reddit.marginalia.nu/");
         bind(Path.class).annotatedWith(Names.named("data-path")).toInstance(Path.of("/var/lib/wmsa/archive.fast/resources"));
     }