Make WMSA_HOME configurable through an environment variable.

marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java

@@ -33,7 +33,7 @@ public abstract class E2ETestBase {
                 .withCopyFileToContainer(jarFile(), "/WMSA.jar")
                 .withCopyFileToContainer(MountableFile.forClasspathResource("init.sh"), "/init.sh")
                 .withExposedPorts(service.port)
-                .withFileSystemBind(modelsPath(), "/var/lib/wmsa/model", BindMode.READ_ONLY)
+                .withFileSystemBind(modelsPath(), "/wmsa/model", BindMode.READ_ONLY)
                 .withNetwork(network)
                 .withNetworkAliases(service.name)
                 .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(service.name)))

marginalia_nu/src/e2e/resources/init.sh

@@ -1,13 +1,15 @@
 #!/bin/bash
 
-mkdir -p /var/lib/wmsa/encyclopedia
+HOME=/wmsa
-mkdir -p /var/lib/wmsa/conf
-mkdir -p /var/lib/wmsa/index/write
-mkdir -p /var/lib/wmsa/index/read
-mkdir -p /backup/work/index-tmp
 
-mkdir -p /var/log/wmsa
+mkdir -p ${HOME}/encyclopedia
-cat > /var/lib/wmsa/suggestions.txt <<EOF
+mkdir -p ${HOME}/conf
+mkdir -p ${HOME}/index/write
+mkdir -p ${HOME}/index/read
+mkdir -p ${HOME}/tmp-slow
+mkdir -p ${HOME}/tmp-fast
+
+cat > ${HOME}/suggestions.txt <<EOF
 state
 three
 while
@@ -22,17 +24,22 @@ many
 year
 EOF
 
-cat > /var/lib/wmsa/conf/disks.properties <<EOF
+cat > ${HOME}/conf/disks.properties <<EOF
-encyclopedia=/var/lib/wmsa/encyclopedia
+encyclopedia=${HOME}/encyclopedia
+
+index-write=${HOME}/index/write
+index-read=${HOME}/index/read
+tmp-slow=${HOME}/tmp-slow
+tmp-fast=${HOME}/tmp-fast
 EOF
 
-cat > /var/lib/wmsa/conf/db.properties <<EOF
+cat > ${HOME}/conf/db.properties <<EOF
   db.user=wmsa
   db.pass=wmsa
   db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true
 EOF
 
-cat > /var/lib/wmsa/conf/ranking-settings.yaml <<EOF
+cat > ${HOME}/conf/ranking-settings.yaml <<EOF
 ---
 retro:
   - "%"
@@ -46,7 +53,7 @@ standard:
   - "%"
 EOF
 
-cat > /var/lib/wmsa/conf/hosts <<EOF
+cat > ${HOME}/conf/hosts <<EOF
 # service-name host-name
 resource-store resource-store
 renderer renderer
@@ -62,4 +69,4 @@ memex memex
 dating dating
 EOF
 
-java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1
+WMSA_HOME=${HOME} java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1

marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java

@@ -1,9 +1,12 @@
 package nu.marginalia.wmsa.configuration;
 
+import nu.marginalia.util.language.conf.LanguageModels;
+
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Optional;
 import java.util.Properties;
 
 public class WmsaHome {
@@ -20,7 +23,9 @@ public class WmsaHome {
     }
 
     public static Path getHomePath() {
-        var ret = Path.of(System.getProperty("WMSA_HOME", DEFAULT));
+        var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse(DEFAULT);
+
+        var ret = Path.of(retStr);
         if (!Files.isDirectory(ret)) {
             throw new IllegalStateException("Could not find WMSA_HOME, either set environment variable or ensure " + DEFAULT + " exists");
         }
@@ -45,26 +50,44 @@ public class WmsaHome {
         return getHomePath().resolve("data").resolve("IP2LOCATION-LITE-DB1.CSV");
     }
 
-    public static Path getDisk(String name) throws IOException {
+    public static Path getDisk(String name) {
-        Path p = Path.of(getDiskProperties().getProperty(name));
+        var pathStr = getDiskProperties().getProperty(name);
+        if (null == pathStr) {
+            throw new RuntimeException("Disk " + name + " was not configured");
+        }
+        Path p = Path.of(pathStr);
         if (!Files.isDirectory(p)) {
-            throw new IOException(name + " does not exist!");
+            throw new RuntimeException("Disk " + name + " does not exist or is not a directory!");
         }
         return p;
     }
 
-    public static Properties getDiskProperties() throws IOException {
+    public static Properties getDiskProperties() {
         Path settingsFile = getHomePath().resolve("conf/disks.properties");
 
-        if (Files.isRegularFile(settingsFile)) {
+        if (!Files.isRegularFile(settingsFile)) {
-            try (var is = Files.newInputStream(settingsFile)) {
+            throw new RuntimeException("Could not find disk settings " + settingsFile);
-                var props = new Properties();
-                props.load(is);
-                return props;
-            }
         }
-        else {
+
-            throw new IOException("Could not find disk settings " + settingsFile);
+        try (var is = Files.newInputStream(settingsFile)) {
+            var props = new Properties();
+            props.load(is);
+            return props;
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
         }
     }
+
+    public static LanguageModels getLanguageModels() {
+        final Path home = getHomePath();
+
+        return new LanguageModels(
+                home.resolve("model/ngrams-generous-emstr.bin"),
+                home.resolve("model/tfreq-new-algo3.bin"),
+                home.resolve("model/opennlp-sentence.bin"),
+                home.resolve("model/English.RDR"),
+                home.resolve("model/English.DICT"),
+                home.resolve("model/opennlp-tok.bin"));
+    }
 }

marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/EdgeAssistantModule.java

@@ -2,6 +2,7 @@ package nu.marginalia.wmsa.edge.assistant;
 
 import com.google.inject.AbstractModule;
 import nu.marginalia.util.language.conf.LanguageModels;
+import nu.marginalia.wmsa.configuration.WmsaHome;
 
 import java.nio.file.Path;
 
@@ -9,14 +10,8 @@ import static com.google.inject.name.Names.named;
 
 public class EdgeAssistantModule extends AbstractModule {
     public void configure() {
-        bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(Path.of("/var/lib/wmsa/suggestions.txt"));
+        bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(WmsaHome.getHomePath().resolve("suggestions.txt"));
-        bind(LanguageModels.class).toInstance(new LanguageModels(
+
-                Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
+        bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
-                Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
-                Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
-                Path.of("/var/lib/wmsa/model/English.RDR"),
-                Path.of("/var/lib/wmsa/model/English.DICT"),
-                Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
-        ));
     }
 }

marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ConverterModule.java

@@ -5,6 +5,7 @@ import com.google.inject.AbstractModule;
 import com.google.inject.name.Names;
 import marcono1234.gson.recordadapter.RecordTypeAdapterFactory;
 import nu.marginalia.util.language.conf.LanguageModels;
+import nu.marginalia.wmsa.configuration.WmsaHome;
 import nu.marginalia.wmsa.edge.model.EdgeCrawlPlan;
 import nu.marginalia.wmsa.edge.model.EdgeDomain;
 import nu.marginalia.wmsa.edge.model.EdgeUrl;
@@ -30,14 +31,7 @@ public class ConverterModule extends AbstractModule {
         bind(Integer.class).annotatedWith(Names.named("max-title-length")).toInstance(128);
         bind(Integer.class).annotatedWith(Names.named("max-summary-length")).toInstance(255);
 
-        bind(LanguageModels.class).toInstance(new LanguageModels(
+        bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
-                Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
-                Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
-                Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
-                Path.of("/var/lib/wmsa/model/English.RDR"),
-                Path.of("/var/lib/wmsa/model/English.DICT"),
-                Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
-        ));
     }
 
     private Gson createGson() {

marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeTablesModule.java

@@ -2,17 +2,18 @@ package nu.marginalia.wmsa.edge.index;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.name.Names;
+import nu.marginalia.wmsa.configuration.WmsaHome;
 
 import java.nio.file.Path;
 
 public class EdgeTablesModule extends AbstractModule {
 
     public void configure() {
-        bind(Path.class).annotatedWith(Names.named("partition-root-slow")).toInstance(Path.of("/var/lib/wmsa/index/write"));
+        bind(Path.class).annotatedWith(Names.named("partition-root-slow")).toInstance(WmsaHome.getDisk("index-write"));
-        bind(Path.class).annotatedWith(Names.named("partition-root-slow-tmp")).toInstance(Path.of("/backup/work/index-tmp/"));
+        bind(Path.class).annotatedWith(Names.named("partition-root-fast")).toInstance(WmsaHome.getDisk("index-read"));
 
-        bind(Path.class).annotatedWith(Names.named("partition-root-fast")).toInstance(Path.of("/var/lib/wmsa/index/read"));
+        bind(Path.class).annotatedWith(Names.named("partition-root-slow-tmp")).toInstance(WmsaHome.getDisk("tmp-slow"));
-        bind(Path.class).annotatedWith(Names.named("tmp-file-dir")).toInstance(Path.of("/var/lib/wmsa/index/read"));
+        bind(Path.class).annotatedWith(Names.named("tmp-file-dir")).toInstance(WmsaHome.getDisk("tmp-fast"));
 
         bind(String.class).annotatedWith(Names.named("edge-writer-page-index-file")).toInstance("page-index.dat");
         bind(String.class).annotatedWith(Names.named("edge-writer-dictionary-file")).toInstance("dictionary.dat");

marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchModule.java

@@ -2,21 +2,12 @@ package nu.marginalia.wmsa.edge.search;
 
 import com.google.inject.AbstractModule;
 import nu.marginalia.util.language.conf.LanguageModels;
-
+import nu.marginalia.wmsa.configuration.WmsaHome;
-import java.nio.file.Path;
 
 public class EdgeSearchModule extends AbstractModule {
 
     public void configure() {
-
+        bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
-        bind(LanguageModels.class).toInstance(new LanguageModels(
-                Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
-                Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
-                Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
-                Path.of("/var/lib/wmsa/model/English.RDR"),
-                Path.of("/var/lib/wmsa/model/English.DICT"),
-                Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
-        ));
     }
 
 }

marginalia_nu/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java

@@ -7,7 +7,6 @@ import java.nio.file.Path;
 
 public class ResourceStoreModule extends AbstractModule {
     public void configure() {
-        bind(String.class).annotatedWith(Names.named("external-url")).toInstance("https://reddit.marginalia.nu/");
         bind(Path.class).annotatedWith(Names.named("data-path")).toInstance(Path.of("/var/lib/wmsa/archive.fast/resources"));
     }