Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make WMSA_HOME configurable through an environment variable.

Browse Source
This commit is contained in:
vlofgren 2022-06-03 13:32:05 +02:00
parent 43fed18063
commit 0e65384781
8 changed files with 70 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java
View File

@ -33,7 +33,7 @@ public abstract class E2ETestBase {
.withCopyFileToContainer(jarFile(), "/WMSA.jar")
.withCopyFileToContainer(MountableFile.forClasspathResource("init.sh"), "/init.sh")
.withExposedPorts(service.port)
.withFileSystemBind(modelsPath(), "/var/lib/wmsa/model", BindMode.READ_ONLY)
.withFileSystemBind(modelsPath(), "/wmsa/model", BindMode.READ_ONLY)
.withNetwork(network)
.withNetworkAliases(service.name)
.withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(service.name)))

33
marginalia_nu/src/e2e/resources/init.sh
View File

@ -1,13 +1,15 @@
#!/bin/bash
mkdir -p /var/lib/wmsa/encyclopedia
mkdir -p /var/lib/wmsa/conf
mkdir -p /var/lib/wmsa/index/write
mkdir -p /var/lib/wmsa/index/read
mkdir -p /backup/work/index-tmp
HOME=/wmsa
mkdir -p /var/log/wmsa
cat > /var/lib/wmsa/suggestions.txt <<EOF
mkdir -p ${HOME}/encyclopedia
mkdir -p ${HOME}/conf
mkdir -p ${HOME}/index/write
mkdir -p ${HOME}/index/read
mkdir -p ${HOME}/tmp-slow
mkdir -p ${HOME}/tmp-fast
cat > ${HOME}/suggestions.txt <<EOF
state
three
while
@ -22,17 +24,22 @@ many
year
EOF
cat > /var/lib/wmsa/conf/disks.properties <<EOF
encyclopedia=/var/lib/wmsa/encyclopedia
cat > ${HOME}/conf/disks.properties <<EOF
encyclopedia=${HOME}/encyclopedia
index-write=${HOME}/index/write
index-read=${HOME}/index/read
tmp-slow=${HOME}/tmp-slow
tmp-fast=${HOME}/tmp-fast
EOF
cat > /var/lib/wmsa/conf/db.properties <<EOF
cat > ${HOME}/conf/db.properties <<EOF
db.user=wmsa
db.pass=wmsa
db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true
EOF
cat > /var/lib/wmsa/conf/ranking-settings.yaml <<EOF
cat > ${HOME}/conf/ranking-settings.yaml <<EOF
---
retro:
- "%"
@ -46,7 +53,7 @@ standard:
- "%"
EOF
cat > /var/lib/wmsa/conf/hosts <<EOF
cat > ${HOME}/conf/hosts <<EOF
# service-name host-name
resource-store resource-store
renderer renderer
@ -62,4 +69,4 @@ memex memex
dating dating
EOF
java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1
WMSA_HOME=${HOME} java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1

49
marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/WmsaHome.java
View File

@ -1,9 +1,12 @@
package nu.marginalia.wmsa.configuration;
import nu.marginalia.util.language.conf.LanguageModels;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import java.util.Properties;
public class WmsaHome {
@ -20,7 +23,9 @@ public class WmsaHome {
}
public static Path getHomePath() {
var ret = Path.of(System.getProperty("WMSA_HOME", DEFAULT));
var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse(DEFAULT);
var ret = Path.of(retStr);
if (!Files.isDirectory(ret)) {
throw new IllegalStateException("Could not find WMSA_HOME, either set environment variable or ensure " + DEFAULT + " exists");
}
@ -45,26 +50,44 @@ public class WmsaHome {
return getHomePath().resolve("data").resolve("IP2LOCATION-LITE-DB1.CSV");
}
public static Path getDisk(String name) throws IOException {
Path p = Path.of(getDiskProperties().getProperty(name));
public static Path getDisk(String name) {
var pathStr = getDiskProperties().getProperty(name);
if (null == pathStr) {
throw new RuntimeException("Disk " + name + " was not configured");
}
Path p = Path.of(pathStr);
if (!Files.isDirectory(p)) {
throw new IOException(name + " does not exist!");
throw new RuntimeException("Disk " + name + " does not exist or is not a directory!");
}
return p;
}
public static Properties getDiskProperties() throws IOException {
public static Properties getDiskProperties() {
Path settingsFile = getHomePath().resolve("conf/disks.properties");
if (Files.isRegularFile(settingsFile)) {
try (var is = Files.newInputStream(settingsFile)) {
var props = new Properties();
props.load(is);
return props;
}
if (!Files.isRegularFile(settingsFile)) {
throw new RuntimeException("Could not find disk settings " + settingsFile);
}
else {
throw new IOException("Could not find disk settings " + settingsFile);
try (var is = Files.newInputStream(settingsFile)) {
var props = new Properties();
props.load(is);
return props;
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
}
public static LanguageModels getLanguageModels() {
final Path home = getHomePath();
return new LanguageModels(
home.resolve("model/ngrams-generous-emstr.bin"),
home.resolve("model/tfreq-new-algo3.bin"),
home.resolve("model/opennlp-sentence.bin"),
home.resolve("model/English.RDR"),
home.resolve("model/English.DICT"),
home.resolve("model/opennlp-tok.bin"));
}
}

13
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/assistant/EdgeAssistantModule.java
View File

@ -2,6 +2,7 @@ package nu.marginalia.wmsa.edge.assistant;
import com.google.inject.AbstractModule;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.configuration.WmsaHome;
import java.nio.file.Path;
@ -9,14 +10,8 @@ import static com.google.inject.name.Names.named;
public class EdgeAssistantModule extends AbstractModule {
public void configure() {
bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(Path.of("/var/lib/wmsa/suggestions.txt"));
bind(LanguageModels.class).toInstance(new LanguageModels(
Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
Path.of("/var/lib/wmsa/model/English.RDR"),
Path.of("/var/lib/wmsa/model/English.DICT"),
Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
));
bind(Path.class).annotatedWith(named("suggestions-file")).toInstance(WmsaHome.getHomePath().resolve("suggestions.txt"));
bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
}
}

10
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/ConverterModule.java
View File

@ -5,6 +5,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.name.Names;
import marcono1234.gson.recordadapter.RecordTypeAdapterFactory;
import nu.marginalia.util.language.conf.LanguageModels;
import nu.marginalia.wmsa.configuration.WmsaHome;
import nu.marginalia.wmsa.edge.model.EdgeCrawlPlan;
import nu.marginalia.wmsa.edge.model.EdgeDomain;
import nu.marginalia.wmsa.edge.model.EdgeUrl;
@ -30,14 +31,7 @@ public class ConverterModule extends AbstractModule {
bind(Integer.class).annotatedWith(Names.named("max-title-length")).toInstance(128);
bind(Integer.class).annotatedWith(Names.named("max-summary-length")).toInstance(255);
bind(LanguageModels.class).toInstance(new LanguageModels(
Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
Path.of("/var/lib/wmsa/model/English.RDR"),
Path.of("/var/lib/wmsa/model/English.DICT"),
Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
));
bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
}
private Gson createGson() {

9
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/index/EdgeTablesModule.java
View File

@ -2,17 +2,18 @@ package nu.marginalia.wmsa.edge.index;
import com.google.inject.AbstractModule;
import com.google.inject.name.Names;
import nu.marginalia.wmsa.configuration.WmsaHome;
import java.nio.file.Path;
public class EdgeTablesModule extends AbstractModule {
public void configure() {
bind(Path.class).annotatedWith(Names.named("partition-root-slow")).toInstance(Path.of("/var/lib/wmsa/index/write"));
bind(Path.class).annotatedWith(Names.named("partition-root-slow-tmp")).toInstance(Path.of("/backup/work/index-tmp/"));
bind(Path.class).annotatedWith(Names.named("partition-root-slow")).toInstance(WmsaHome.getDisk("index-write"));
bind(Path.class).annotatedWith(Names.named("partition-root-fast")).toInstance(WmsaHome.getDisk("index-read"));
bind(Path.class).annotatedWith(Names.named("partition-root-fast")).toInstance(Path.of("/var/lib/wmsa/index/read"));
bind(Path.class).annotatedWith(Names.named("tmp-file-dir")).toInstance(Path.of("/var/lib/wmsa/index/read"));
bind(Path.class).annotatedWith(Names.named("partition-root-slow-tmp")).toInstance(WmsaHome.getDisk("tmp-slow"));
bind(Path.class).annotatedWith(Names.named("tmp-file-dir")).toInstance(WmsaHome.getDisk("tmp-fast"));
bind(String.class).annotatedWith(Names.named("edge-writer-page-index-file")).toInstance("page-index.dat");
bind(String.class).annotatedWith(Names.named("edge-writer-dictionary-file")).toInstance("dictionary.dat");

13
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/search/EdgeSearchModule.java
View File

@ -2,21 +2,12 @@ package nu.marginalia.wmsa.edge.search;
import com.google.inject.AbstractModule;
import nu.marginalia.util.language.conf.LanguageModels;
import java.nio.file.Path;
import nu.marginalia.wmsa.configuration.WmsaHome;
public class EdgeSearchModule extends AbstractModule {
public void configure() {
bind(LanguageModels.class).toInstance(new LanguageModels(
Path.of("/var/lib/wmsa/model/ngrams-generous-emstr.bin"),
Path.of("/var/lib/wmsa/model/tfreq-new-algo3.bin"),
Path.of("/var/lib/wmsa/model/opennlp-sentence.bin"),
Path.of("/var/lib/wmsa/model/English.RDR"),
Path.of("/var/lib/wmsa/model/English.DICT"),
Path.of("/var/lib/wmsa/model/opennlp-tok.bin")
));
bind(LanguageModels.class).toInstance(WmsaHome.getLanguageModels());
}
}

1
marginalia_nu/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java
View File

@ -7,7 +7,6 @@ import java.nio.file.Path;
public class ResourceStoreModule extends AbstractModule {
public void configure() {
bind(String.class).annotatedWith(Names.named("external-url")).toInstance("https://reddit.marginalia.nu/");
bind(Path.class).annotatedWith(Names.named("data-path")).toInstance(Path.of("/var/lib/wmsa/archive.fast/resources"));
}