Merge pull request 'Memex refactored' (#32) from master into release

Reviewed-on: https://git.marginalia.nu/marginalia/marginalia.nu/pulls/32

marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/E2ETestBase.java

@@ -43,6 +43,22 @@ public abstract class E2ETestBase {
                         .withReadTimeout(Duration.ofSeconds(15)))
                 ;
     }
+    public static GenericContainer<?> forService(ServiceDescriptor service, GenericContainer<?> mariaDB, String setupScript) {
+        return new GenericContainer<>("openjdk:17-alpine")
+                .dependsOn(mariaDB)
+                .withCopyFileToContainer(jarFile(), "/WMSA.jar")
+                .withCopyFileToContainer(MountableFile.forClasspathResource(setupScript), "/" + setupScript)
+                .withExposedPorts(service.port)
+                .withFileSystemBind(modelsPath(), "/wmsa/model", BindMode.READ_ONLY)
+                .withNetwork(network)
+                .withNetworkAliases(service.name)
+                .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(service.name)))
+                .withCommand("sh", setupScript, service.name)
+                .waitingFor(Wait.forHttp("/internal/ping")
+                        .forPort(service.port)
+                        .withReadTimeout(Duration.ofSeconds(15)))
+                ;
+    }
 
     public static MountableFile jarFile() {
         Path cwd = Path.of(System.getProperty("user.dir"));

marginalia_nu/src/e2e/java/nu/marginalia/wmsa/edge/MemexE2ETest.java

@@ -0,0 +1,95 @@
+package nu.marginalia.wmsa.edge;
+
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import okhttp3.OkHttpClient;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.mariadb.jdbc.Driver;
+import org.openqa.selenium.OutputType;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.*;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.MountableFile;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.util.concurrent.TimeUnit;
+
+import static nu.marginalia.wmsa.configuration.ServiceDescriptor.AUTH;
+import static nu.marginalia.wmsa.configuration.ServiceDescriptor.MEMEX;
+
+@Tag("e2e")
+@Testcontainers
+public class MemexE2ETest extends E2ETestBase {
+    @Container
+    public MariaDBContainer<?> mariaDB = getMariaDBContainer();
+
+    @Container
+    public GenericContainer<?> auth = forService(AUTH, mariaDB);
+
+    @Container
+    public GenericContainer<?> memexContainer =  forService(MEMEX, mariaDB, "memex.sh")
+            .withClasspathResourceMapping("/memex", "/memex", BindMode.READ_ONLY);
+
+    @Container
+    public NginxContainer<?> proxyNginx = new NginxContainer<>("nginx:stable")
+            .dependsOn(auth)
+            .dependsOn(memexContainer)
+            .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger("nginx")))
+            .withCopyFileToContainer(MountableFile.forClasspathResource("nginx/memex.conf"), "/etc/nginx/conf.d/default.conf")
+            .withNetwork(network)
+            .withNetworkAliases("proxyNginx");
+
+    @Container
+    public BrowserWebDriverContainer<?> chrome = new BrowserWebDriverContainer<>()
+            .withNetwork(network)
+            .withCapabilities(new ChromeOptions());
+
+    private Gson gson = new GsonBuilder().create();
+    private OkHttpClient httpClient = new OkHttpClient.Builder()
+            .connectTimeout(100, TimeUnit.MILLISECONDS)
+            .readTimeout(6000, TimeUnit.SECONDS)
+            .retryOnConnectionFailure(true)
+            .followRedirects(true)
+            .build();
+
+    @Test
+    public void run() throws IOException, InterruptedException {
+        Thread.sleep(10_000);
+        new Driver();
+
+        var driver = chrome.getWebDriver();
+
+        driver.get("http://proxyNginx/");
+        Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("frontpage"));
+
+        driver.get("http://proxyNginx/log/");
+        Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("log"));
+
+        driver.get("http://proxyNginx/log/a.gmi");
+        Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("log-a.gmi"));
+
+        driver.get("http://proxyNginx/log/b.gmi");
+        Files.move(driver.getScreenshotAs(OutputType.FILE).toPath(), screenshotFilename("log-b.gmi"));
+    }
+
+    private static Path screenshotFilename(String operation) throws IOException {
+        var path = Path.of(System.getProperty("user.dir")).resolve("build/test/e2e/");
+        Files.createDirectories(path);
+
+        String name = String.format("test-%s-%s.png", operation, LocalDateTime.now());
+        path = path.resolve(name);
+
+        System.out.println("Screenshot in " + path);
+        return path;
+    }
+
+
+}

marginalia_nu/src/e2e/resources/init.sh

@@ -69,4 +69,5 @@ memex memex
 dating dating
 EOF
 
+echo "*** Starting $1"
 WMSA_HOME=${HOME} java -Dsmall-ram=TRUE -Dservice-host=0.0.0.0 -jar /WMSA.jar start $1

marginalia_nu/src/e2e/resources/memex.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+HOME=/wmsa
+
+mkdir -p ${HOME}/conf
+
+cat > ${HOME}/conf/db.properties <<EOF
+  db.user=wmsa
+  db.pass=wmsa
+  db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true
+EOF
+
+cat > ${HOME}/conf/hosts <<EOF
+# service-name host-name
+resource-store resource-store
+renderer renderer
+auth auth
+api api
+smhi-scraper smhi-scraper
+podcast-scraper podcast-scraper
+edge-index edge-index
+edge-search edge-search
+encyclopedia encyclopedia
+edge-assistant edge-assistant
+memex memex
+dating dating
+EOF
+
+mkdir -p /memex /gmi /html
+
+echo "*** Starting $1"
+WMSA_HOME=${HOME} java \
+    -Dmemex-root=/memex\
+    -Dmemex-html-resources=/html\
+    -Dmemex-gmi-resources=/gmi\
+    -Dmemex-disable-git=TRUE\
+    -Dmemex-disable-gemini=TRUE\
+    -Dservice-host=0.0.0.0\
+    -jar /WMSA.jar start $1

marginalia_nu/src/e2e/resources/memex/index.gmi

@@ -0,0 +1,6 @@
+# Memex Index
+
+Nemo enim ipsam voluptatem, quia voluptas sit, aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos,
+qui ratione voluptatem sequi nesciunt, neque porro quisquam est, qui dolorem ipsum, quia dolor sit amet consectetur
+adipiscing velit, sed quia non numquam do eius modi tempora incididunt, ut labore et dolore magnam aliquam quaerat
+voluptatem.

marginalia_nu/src/e2e/resources/memex/log/a.gmi

@@ -0,0 +1,7 @@
+# A
+
+AAAAAAAAAAA
+AAAA
+
+
+AAAAA

marginalia_nu/src/e2e/resources/memex/log/b.gmi

@@ -0,0 +1,6 @@
+# B
+
+BBBB
+BBBB
+BBBB
+BBBB

marginalia_nu/src/e2e/resources/memex/log/index.gmi

@@ -0,0 +1,7 @@
+# Log
+
+Log of stuff
+
+%%%FEED
+%%%LISTING
+

marginalia_nu/src/e2e/resources/nginx/memex.conf

@@ -0,0 +1,27 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  nginx;
+
+    location ~* \.(gmi|png)$ {
+
+        types {
+            text/html gmi;
+            text/html png;
+        }
+
+        proxy_pass http://memex:5030$uri;
+    }
+    location ~* feed\.xml {
+        types {
+            application/xml+atom xml;
+        }
+
+        proxy_pass http://memex:5030$uri;
+    }
+    location / {
+        proxy_pass http://memex:5030;
+    }
+
+
+}

marginalia_nu/src/main/java/nu/marginalia/gemini/GeminiService.java

@@ -1,164 +1,7 @@
 package nu.marginalia.gemini;
 
-import com.google.inject.Inject;
+public interface GeminiService {
-import com.google.inject.Singleton;
+    String DEFAULT_FILENAME = "index.gmi";
-import com.google.inject.name.Named;
-import nu.marginalia.gemini.io.GeminiConnection;
-import nu.marginalia.gemini.io.GeminiSSLSetUp;
-import nu.marginalia.gemini.io.GeminiStatusCode;
-import nu.marginalia.gemini.io.GeminiUserException;
-import nu.marginalia.gemini.plugins.BareStaticPagePlugin;
-import nu.marginalia.gemini.plugins.Plugin;
-import nu.marginalia.gemini.plugins.SearchPlugin;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLServerSocket;
-import javax.net.ssl.SSLServerSocketFactory;
-import javax.net.ssl.SSLSocket;
-import java.io.IOException;
-import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Optional;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-
-@Singleton
-public class GeminiService {
-
-    public static final String DEFAULT_FILENAME = "index.gmi";
-    public final Path serverRoot;
-
-    private final Logger logger = LoggerFactory.getLogger("GeminiServer");
-    private final Executor pool = Executors.newFixedThreadPool(32);
-    private final SSLServerSocket serverSocket;
-
-    private final Plugin[] plugins;
-    private final BadBotList badBotList = BadBotList.INSTANCE;
-
-    @Inject
-    public GeminiService(@Named("gemini-server-root") Path serverRoot,
-                         @Named("gemini-server-port") Integer port,
-                         GeminiSSLSetUp sslSetUp,
-                         BareStaticPagePlugin pagePlugin,
-                         SearchPlugin searchPlugin) throws Exception {
-        this.serverRoot = serverRoot;
-        logger.info("Setting up crypto");
-        final SSLServerSocketFactory socketFactory = sslSetUp.getServerSocketFactory();
-
-        serverSocket = (SSLServerSocket) socketFactory.createServerSocket(port /* 1965 */);
-        serverSocket.setEnabledCipherSuites(socketFactory.getSupportedCipherSuites());
-        serverSocket.setEnabledProtocols(new String[] {"TLSv1.3", "TLSv1.2"});
-
-        logger.info("Verifying setup");
-        if (!Files.exists(this.serverRoot)) {
-            logger.error("Could not find SERVER_ROOT {}", this.serverRoot);
-            System.exit(255);
-        }
-
-        plugins = new Plugin[] {
-                pagePlugin,
-                searchPlugin
-        };
-    }
-
-    public void run()  {
-        logger.info("Awaiting connections");
-
-        try {
-            for (; ; ) {
-                SSLSocket connection = (SSLSocket) serverSocket.accept();
-                connection.setSoTimeout(10_000);
-
-                if (!badBotList.isAllowed(connection.getInetAddress())) {
-                    connection.close();
-                } else {
-                    pool.execute(() -> serve(connection));
-                }
-            }
-        }
-        catch (IOException ex) {
-            logger.error("IO Exception in gemini server", ex);
-        }
-    }
-
-    private void serve(SSLSocket socket) {
-        final GeminiConnection connection;
-        try {
-            connection = new GeminiConnection(socket);
-        }
-        catch (IOException ex) {
-            logger.error("Failed to create connection object", ex);
-            return;
-        }
-
-        try {
-            handleRequest(connection);
-        }
-        catch (GeminiUserException ex) {
-            errorResponse(connection, ex.getMessage());
-        }
-        catch (SSLException ex) {
-            logger.error(connection.getAddress() + " SSL error");
-            connection.close();
-        }
-        catch (Exception ex) {
-            errorResponse(connection, "Error");
-            logger.error(connection.getAddress(), ex);
-        }
-        finally {
-            connection.close();
-        }
-    }
-
-    private void errorResponse(GeminiConnection connection, String message) {
-        if (connection.isConnected()) {
-            try {
-                logger.error("=> " + connection.getAddress(), message);
-                connection.writeStatusLine(GeminiStatusCode.ERROR_PERMANENT, message);
-            }
-            catch (IOException ex) {
-                logger.error("Exception while sending error", ex);
-            }
-        }
-    }
-
-    private void handleRequest(GeminiConnection connection) throws Exception {
-
-        final String address = connection.getAddress();
-        logger.info("Connect: " + address);
-
-        final Optional<URI> maybeUri = connection.readUrl();
-        if (maybeUri.isEmpty()) {
-            logger.info("Done: {}", address);
-            return;
-        }
-
-        final URI uri = maybeUri.get();
-        logger.info("Request {}", uri);
-
-        if (!uri.getScheme().equals("gemini")) {
-            throw new GeminiUserException("Unsupported protocol");
-        }
-
-        servePage(connection, uri);
-        logger.info("Done: {}", address);
-    }
-
-    private void servePage(GeminiConnection connection, URI url) throws IOException {
-        String path = url.getPath();
-
-        for (Plugin p : plugins) {
-            if (p.serve(url, connection)) {
-                return;
-            }
-        }
-
-        logger.error("FileNotFound {}", path);
-        connection.writeStatusLine(GeminiStatusCode.ERROR_TEMPORARY, "No such file");
-    }
-
 
+    void run();
 }

marginalia_nu/src/main/java/nu/marginalia/gemini/GeminiServiceDummy.java

@@ -0,0 +1,10 @@
+package nu.marginalia.gemini;
+
+import com.google.inject.Singleton;
+
+@Singleton
+public class GeminiServiceDummy implements GeminiService {
+    @Override
+    public void run() {
+    }
+}

marginalia_nu/src/main/java/nu/marginalia/gemini/GeminiServiceImpl.java

@@ -0,0 +1,164 @@
+package nu.marginalia.gemini;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+import nu.marginalia.gemini.io.GeminiConnection;
+import nu.marginalia.gemini.io.GeminiSSLSetUp;
+import nu.marginalia.gemini.io.GeminiStatusCode;
+import nu.marginalia.gemini.io.GeminiUserException;
+import nu.marginalia.gemini.plugins.BareStaticPagePlugin;
+import nu.marginalia.gemini.plugins.Plugin;
+import nu.marginalia.gemini.plugins.SearchPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLServerSocketFactory;
+import javax.net.ssl.SSLSocket;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+@Singleton
+public class GeminiServiceImpl implements GeminiService {
+
+    public final Path serverRoot;
+
+    private final Logger logger = LoggerFactory.getLogger(getClass().getSimpleName());
+    private final Executor pool = Executors.newFixedThreadPool(32);
+    private final SSLServerSocket serverSocket;
+
+    private final Plugin[] plugins;
+    private final BadBotList badBotList = BadBotList.INSTANCE;
+
+    @Inject
+    public GeminiServiceImpl(@Named("gemini-server-root") Path serverRoot,
+                             @Named("gemini-server-port") Integer port,
+                             GeminiSSLSetUp sslSetUp,
+                             BareStaticPagePlugin pagePlugin,
+                             SearchPlugin searchPlugin) throws Exception {
+        this.serverRoot = serverRoot;
+        logger.info("Setting up crypto");
+        final SSLServerSocketFactory socketFactory = sslSetUp.getServerSocketFactory();
+
+        serverSocket = (SSLServerSocket) socketFactory.createServerSocket(port /* 1965 */);
+        serverSocket.setEnabledCipherSuites(socketFactory.getSupportedCipherSuites());
+        serverSocket.setEnabledProtocols(new String[] {"TLSv1.3", "TLSv1.2"});
+
+        logger.info("Verifying setup");
+        if (!Files.exists(this.serverRoot)) {
+            logger.error("Could not find SERVER_ROOT {}", this.serverRoot);
+            System.exit(255);
+        }
+
+        plugins = new Plugin[] {
+                pagePlugin,
+                searchPlugin
+        };
+    }
+
+    @Override
+    public void run()  {
+        logger.info("Awaiting connections");
+
+        try {
+            for (;;) {
+                SSLSocket connection = (SSLSocket) serverSocket.accept();
+                connection.setSoTimeout(10_000);
+
+                if (!badBotList.isAllowed(connection.getInetAddress())) {
+                    connection.close();
+                } else {
+                    pool.execute(() -> serve(connection));
+                }
+            }
+        }
+        catch (IOException ex) {
+            logger.error("IO Exception in gemini server", ex);
+        }
+    }
+
+    private void serve(SSLSocket socket) {
+        final GeminiConnection connection;
+        try {
+            connection = new GeminiConnection(socket);
+        }
+        catch (IOException ex) {
+            logger.error("Failed to create connection object", ex);
+            return;
+        }
+
+        try {
+            handleRequest(connection);
+        }
+        catch (GeminiUserException ex) {
+            errorResponse(connection, ex.getMessage());
+        }
+        catch (SSLException ex) {
+            logger.error(connection.getAddress() + " SSL error");
+            connection.close();
+        }
+        catch (Exception ex) {
+            errorResponse(connection, "Error");
+            logger.error(connection.getAddress(), ex);
+        }
+        finally {
+            connection.close();
+        }
+    }
+
+    private void errorResponse(GeminiConnection connection, String message) {
+        if (connection.isConnected()) {
+            try {
+                logger.error("=> " + connection.getAddress(), message);
+                connection.writeStatusLine(GeminiStatusCode.ERROR_PERMANENT, message);
+            }
+            catch (IOException ex) {
+                logger.error("Exception while sending error", ex);
+            }
+        }
+    }
+
+    private void handleRequest(GeminiConnection connection) throws Exception {
+
+        final String address = connection.getAddress();
+        logger.info("Connect: " + address);
+
+        final Optional<URI> maybeUri = connection.readUrl();
+        if (maybeUri.isEmpty()) {
+            logger.info("Done: {}", address);
+            return;
+        }
+
+        final URI uri = maybeUri.get();
+        logger.info("Request {}", uri);
+
+        if (!uri.getScheme().equals("gemini")) {
+            throw new GeminiUserException("Unsupported protocol");
+        }
+
+        servePage(connection, uri);
+        logger.info("Done: {}", address);
+    }
+
+    private void servePage(GeminiConnection connection, URI url) throws IOException {
+        String path = url.getPath();
+
+        for (Plugin p : plugins) {
+            if (p.serve(url, connection)) {
+                return;
+            }
+        }
+
+        logger.error("FileNotFound {}", path);
+        connection.writeStatusLine(GeminiStatusCode.ERROR_TEMPORARY, "No such file");
+    }
+
+
+}

marginalia_nu/src/main/java/nu/marginalia/gemini/plugins/BareStaticPagePlugin.java

@@ -2,6 +2,7 @@ package nu.marginalia.gemini.plugins;
 
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
+import nu.marginalia.gemini.GeminiService;
 import nu.marginalia.gemini.io.GeminiConnection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -11,8 +12,6 @@ import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import static nu.marginalia.gemini.GeminiService.DEFAULT_FILENAME;
-
 public class BareStaticPagePlugin implements Plugin {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
@@ -43,8 +42,8 @@ public class BareStaticPagePlugin implements Plugin {
     private Path getServerPath(String requestPath) {
         final Path serverPath = Path.of(geminiServerRoot + requestPath);
 
-        if (Files.isDirectory(serverPath) && Files.isRegularFile(serverPath.resolve(DEFAULT_FILENAME))) {
+        if (Files.isDirectory(serverPath) && Files.isRegularFile(serverPath.resolve(GeminiService.DEFAULT_FILENAME))) {
-            return serverPath.resolve(DEFAULT_FILENAME);
+            return serverPath.resolve(GeminiService.DEFAULT_FILENAME);
         }
 
         return serverPath;

marginalia_nu/src/main/java/nu/marginalia/wmsa/auth/AuthService.java

@@ -1,6 +1,5 @@
 package nu.marginalia.wmsa.auth;
 
-import com.github.jknack.handlebars.internal.Files;
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 import nu.marginalia.wmsa.auth.model.LoginFormModel;
@@ -14,11 +13,12 @@ import spark.Request;
 import spark.Response;
 import spark.Spark;
 
-import java.io.FileReader;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.UUID;
 
 import static spark.Spark.*;
 
@@ -40,11 +40,8 @@ public class AuthService extends Service {
 
         super(ip, port, initialization, metricsServer);
 
-        try (var is = new FileReader(topSecretPasswordFile.toFile())) {
+        password = initPassword(topSecretPasswordFile);
-            password = Files.read(is);
+
-        } catch (IOException e) {
-            logger.error("Could not read password from file " + topSecretPasswordFile, e);
-        }
         loginFormRenderer = rendererFactory.renderer("auth/login");
 
         Spark.path("public/api", () -> {
@@ -60,6 +57,18 @@ public class AuthService extends Service {
         });
     }
 
+    private String initPassword(Path topSecretPasswordFile) {
+        if (Files.exists(topSecretPasswordFile)) {
+            try {
+                return Files.readString(topSecretPasswordFile);
+            } catch (IOException e) {
+                logger.error("Could not read password from file " + topSecretPasswordFile, e);
+            }
+        }
+        logger.error("Setting random password");
+        return UUID.randomUUID().toString();
+    }
+
     private Object loginForm(Request request, Response response) {
         String redir = Objects.requireNonNull(request.queryParams("redirect"));
         String service = Objects.requireNonNull(request.queryParams("service"));

marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/ServiceDescriptor.java

@@ -1,7 +1,7 @@
 package nu.marginalia.wmsa.configuration;
 
-import nu.marginalia.wmsa.auth.AuthMain;
 import nu.marginalia.wmsa.api.ApiMain;
+import nu.marginalia.wmsa.auth.AuthMain;
 import nu.marginalia.wmsa.configuration.command.Command;
 import nu.marginalia.wmsa.configuration.command.ListCommand;
 import nu.marginalia.wmsa.configuration.command.StartCommand;
@@ -35,7 +35,7 @@ public enum ServiceDescriptor {
     EDGE_SEARCH("edge-search", 5023, EdgeSearchMain.class),
     EDGE_ASSISTANT("edge-assistant", 5025, EdgeAssistantMain.class),
 
-    EDGE_MEMEX("memex", 5030, MemexMain.class),
+    MEMEX("memex", 5030, MemexMain.class),
 
     ENCYCLOPEDIA("encyclopedia", 5040, EncyclopediaMain.class),
 
@@ -79,7 +79,6 @@ public enum ServiceDescriptor {
     }
 
     public static void main(String... args) {
-
         MainMapLookup.setMainArguments(args);
         Map<String, Command> functions = Stream.of(new ListCommand(),
                 new StartCommand(),

marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/command/StartCommand.java

@@ -16,7 +16,6 @@ public class StartCommand extends Command {
             System.err.println("Usage: start service-descriptor");
             System.exit(255);
         }
-
         var mainMethod = getKind(args[1]).mainClass.getMethod("main", String[].class);
         String[] args2 = Arrays.copyOfRange(args, 2, args.length);
         mainMethod.invoke(null, (Object) args2);

marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/server/Service.java

@@ -37,7 +37,7 @@ public class Service {
 
     private static volatile boolean initialized = false;
 
-    public Service(String ip, int port, Initialization initialization, MetricsServer metricsServer) {
+    public Service(String ip, int port, Initialization initialization, MetricsServer metricsServer, Runnable configureStaticFiles) {
         this.initialization = initialization;
 
         serviceName = System.getProperty("service-name");
@@ -51,8 +51,7 @@ public class Service {
 
             logger.info("{} Listening to {}:{}", getClass().getSimpleName(), ip == null ? "" : ip, port);
 
-            Spark.staticFiles.expireTime(3600);
+            configureStaticFiles.run();
-            Spark.staticFiles.header("Cache-control", "public");
 
             Spark.before(this::filterPublicRequests);
             Spark.before(this::auditRequestIn);
@@ -66,24 +65,35 @@ public class Service {
         }
     }
 
+    public Service(String ip, int port, Initialization initialization, MetricsServer metricsServer) {
+        this(ip, port, initialization, metricsServer, () -> {
+            // configureStaticFiles can't be an overridable method in Service because it may
+            // need to depend on parameters to the constructor, and super-constructors
+            // must run first
+            Spark.staticFiles.expireTime(3600);
+            Spark.staticFiles.header("Cache-control", "public");
+        });
+    }
+
     private void filterPublicRequests(Request request, Response response) {
-        if (null != request.headers("X-Public")) {
+        if (null == request.headers("X-Public")) {
-
+            return;
-            String context = Optional
-                            .ofNullable(request.headers("X-Context"))
-                            .orElseGet(request::ip);
-
-            if (!request.pathInfo().startsWith("/public/")) {
-                logger.warn(httpMarker, "External connection to internal API: {} -> {} {}", context, request.requestMethod(), request.pathInfo());
-                Spark.halt(HttpStatus.SC_FORBIDDEN);
-            }
-
-            String url = request.pathInfo();
-            if (request.queryString() != null) {
-                url = url + "?" + request.queryString();
-            }
-            logger.info(httpMarker, "PUBLIC {}: {} {}", Context.fromRequest(request).getIpHash().orElse("?"), request.requestMethod(), url);
         }
+
+        String context = Optional
+                        .ofNullable(request.headers("X-Context"))
+                        .orElseGet(request::ip);
+
+        if (!request.pathInfo().startsWith("/public/")) {
+            logger.warn(httpMarker, "External connection to internal API: {} -> {} {}", context, request.requestMethod(), request.pathInfo());
+            Spark.halt(HttpStatus.SC_FORBIDDEN);
+        }
+
+        String url = request.pathInfo();
+        if (request.queryString() != null) {
+            url = url + "?" + request.queryString();
+        }
+        logger.info(httpMarker, "PUBLIC {}: {} {}", Context.fromRequest(request).getIpHash().orElse("?"), request.requestMethod(), url);
     }
 
     private Object isInitialized(Request request, Response response) {

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/Memex.java

@@ -6,9 +6,9 @@ import com.google.inject.name.Named;
 import io.reactivex.rxjava3.schedulers.Schedulers;
 import nu.marginalia.gemini.GeminiService;
 import nu.marginalia.gemini.gmi.GemtextDatabase;
+import nu.marginalia.gemini.gmi.GemtextDocument;
 import nu.marginalia.util.graphics.dithering.FloydSteinbergDither;
 import nu.marginalia.util.graphics.dithering.Palettes;
-import nu.marginalia.gemini.gmi.GemtextDocument;
 import nu.marginalia.wmsa.memex.change.GemtextTombstoneUpdateCaclulator;
 import nu.marginalia.wmsa.memex.model.MemexImage;
 import nu.marginalia.wmsa.memex.model.MemexNode;
@@ -16,7 +16,7 @@ import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
 import nu.marginalia.wmsa.memex.renderer.MemexRendererers;
 import nu.marginalia.wmsa.memex.system.MemexFileSystemMonitor;
 import nu.marginalia.wmsa.memex.system.MemexFileWriter;
-import nu.marginalia.wmsa.memex.system.MemexGitRepo;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/MemexConfigurationModule.java

@@ -5,23 +5,59 @@ import com.google.inject.Inject;
 import com.google.inject.Provider;
 import com.google.inject.name.Named;
 import com.google.inject.name.Names;
+import lombok.SneakyThrows;
+import nu.marginalia.gemini.GeminiService;
+import nu.marginalia.gemini.GeminiServiceDummy;
+import nu.marginalia.gemini.GeminiServiceImpl;
 import nu.marginalia.wmsa.memex.system.MemexFileWriter;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepo;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepoDummy;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepoImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.nio.file.Path;
 
 public class MemexConfigurationModule extends AbstractModule {
+    private static final Logger logger = LoggerFactory.getLogger(MemexConfigurationModule.class);
+
+    private static final String MEMEX_ROOT_PROPERTY = System.getProperty("memex-root", "/var/lib/wmsa/memex");
+    private static final String MEMEX_HTML_PROPERTY = System.getProperty("memex-html-resources", "/var/lib/wmsa/memex-html");
+    private static final String MEMEX_GMI_PROPERTY = System.getProperty("memex-gmi-resources", "/var/lib/wmsa/memex-gmi");
+
+    private static final boolean MEMEX_DISABLE_GIT    = Boolean.getBoolean("memex-disable-git");
+    private static final boolean MEMEX_DISABLE_GEMINI = Boolean.getBoolean("memex-disable-gemini");
+
+    @SneakyThrows
+    public MemexConfigurationModule() {
+        Thread.sleep(100);
+    }
+
     public void configure() {
-        bind(Path.class).annotatedWith(Names.named("memex-root")).toInstance(Path.of("/var/lib/wmsa/memex"));
+        bind(Path.class).annotatedWith(Names.named("memex-root")).toInstance(Path.of(MEMEX_ROOT_PROPERTY));
-        bind(Path.class).annotatedWith(Names.named("memex-html-resources")).toInstance(Path.of("/var/lib/wmsa/memex-html"));
+        bind(Path.class).annotatedWith(Names.named("memex-html-resources")).toInstance(Path.of(MEMEX_HTML_PROPERTY));
-        bind(Path.class).annotatedWith(Names.named("memex-gmi-resources")).toInstance(Path.of("/var/lib/wmsa/memex-gmi"));
+        bind(Path.class).annotatedWith(Names.named("memex-gmi-resources")).toInstance(Path.of(MEMEX_GMI_PROPERTY));
+
         bind(String.class).annotatedWith(Names.named("tombestone-special-file")).toInstance("/special/tombstone.gmi");
         bind(String.class).annotatedWith(Names.named("redirects-special-file")).toInstance("/special/redirect.gmi");
 
+        switchImpl(MemexGitRepo.class, MEMEX_DISABLE_GIT, MemexGitRepoDummy.class, MemexGitRepoImpl.class);
+        switchImpl(GeminiService.class, MEMEX_DISABLE_GEMINI, GeminiServiceDummy.class, GeminiServiceImpl.class);
+
         bind(MemexFileWriter.class).annotatedWith(Names.named("html")).toProvider(MemexHtmlWriterProvider.class);
         bind(MemexFileWriter.class).annotatedWith(Names.named("gmi")).toProvider(MemexGmiWriterProvider.class);
     }
 
-
+    <T> void switchImpl(Class<T> impl, boolean param, Class<? extends T> ifEnabled, Class<? extends T> ifDisabled) {
+        final Class<? extends T> choice;
+        if (param) {
+            choice = ifEnabled;
+        }
+        else {
+            choice = ifDisabled;
+        }
+        bind(impl).to(choice).asEagerSingleton();
+    }
 
     public static class MemexHtmlWriterProvider implements Provider<MemexFileWriter> {
         private final Path path;

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/MemexMain.java

@@ -18,7 +18,7 @@ public class MemexMain extends MainClass {
     }
 
     public static void main(String... args) {
-        init(ServiceDescriptor.EDGE_MEMEX, args);
+        init(ServiceDescriptor.MEMEX, args);
 
         Injector injector = Guice.createInjector(
                 new MemexConfigurationModule(),

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/MemexService.java

@@ -3,6 +3,7 @@ package nu.marginalia.wmsa.memex;
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 import lombok.SneakyThrows;
+import nu.marginalia.gemini.gmi.GemtextDocument;
 import nu.marginalia.gemini.gmi.renderer.GemtextRendererFactory;
 import nu.marginalia.wmsa.auth.client.AuthClient;
 import nu.marginalia.wmsa.configuration.server.Context;
@@ -10,12 +11,11 @@ import nu.marginalia.wmsa.configuration.server.Initialization;
 import nu.marginalia.wmsa.configuration.server.MetricsServer;
 import nu.marginalia.wmsa.configuration.server.Service;
 import nu.marginalia.wmsa.memex.change.GemtextMutation;
-import nu.marginalia.gemini.gmi.GemtextDocument;
 import nu.marginalia.wmsa.memex.change.update.GemtextDocumentUpdateCalculator;
-import nu.marginalia.wmsa.memex.renderer.MemexHtmlRenderer;
 import nu.marginalia.wmsa.memex.model.MemexNodeHeadingId;
 import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
 import nu.marginalia.wmsa.memex.model.render.*;
+import nu.marginalia.wmsa.memex.renderer.MemexHtmlRenderer;
 import org.apache.http.HttpStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,9 +49,18 @@ public class MemexService extends Service {
                         MemexHtmlRenderer renderer,
                         AuthClient authClient,
                         Initialization initialization,
-                        MetricsServer metricsServer) {
+                        MetricsServer metricsServer,
+                        @Named("memex-html-resources") Path memexHtmlDir
+                        ) {
 
-        super(ip, port, initialization, metricsServer);
+        super(ip, port, initialization, metricsServer, () -> {
+            staticFiles.externalLocation(memexHtmlDir.toString());
+            staticFiles.disableMimeTypeGuessing();
+            staticFiles.registerMimeType("gmi", "text/html");
+            staticFiles.registerMimeType("png", "text/html");
+            staticFiles.expireTime(60);
+            staticFiles.header("Cache-control", "public,proxy-revalidate");
+        });
 
         this.updateCalculator = updateCalculator;
         this.memex = memex;

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/client/MemexApiClient.java

@@ -8,7 +8,7 @@ import nu.marginalia.wmsa.configuration.ServiceDescriptor;
 public class MemexApiClient extends AbstractDynamicClient {
     @Inject
     public MemexApiClient() {
-        super(ServiceDescriptor.EDGE_MEMEX);
+        super(ServiceDescriptor.MEMEX);
     }
 
 }

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/system/MemexSourceFileSystem.java

@@ -4,11 +4,15 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
 import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.nio.file.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 
 @Singleton
 public class MemexSourceFileSystem {

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/system/git/MemexGitRepo.java

@@ -0,0 +1,15 @@
+package nu.marginalia.wmsa.memex.system.git;
+
+import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
+
+public interface MemexGitRepo {
+    void pull();
+
+    void remove(MemexNodeUrl url);
+
+    void add(MemexNodeUrl url);
+
+    void update(MemexNodeUrl url);
+
+    void rename(MemexNodeUrl src, MemexNodeUrl dst);
+}

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/system/git/MemexGitRepoDummy.java

@@ -0,0 +1,36 @@
+package nu.marginalia.wmsa.memex.system.git;
+
+import com.google.inject.Singleton;
+import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Singleton
+public class MemexGitRepoDummy implements MemexGitRepo {
+    private static final Logger logger = LoggerFactory.getLogger(MemexGitRepoDummy.class);
+
+    @Override
+    public void pull() {
+        logger.info("Would perform a pull here");
+    }
+
+    @Override
+    public void remove(MemexNodeUrl url) {
+        logger.info("Would perform a remove here");
+    }
+
+    @Override
+    public void add(MemexNodeUrl url) {
+        logger.info("Would perform an add here");
+    }
+
+    @Override
+    public void update(MemexNodeUrl url) {
+        logger.info("Would perform an update here");
+    }
+
+    @Override
+    public void rename(MemexNodeUrl src, MemexNodeUrl dst) {
+        logger.info("Would perform a rename here");
+    }
+}

marginalia_nu/src/main/java/nu/marginalia/wmsa/memex/system/git/MemexGitRepoImpl.java

@@ -1,4 +1,4 @@
-package nu.marginalia.wmsa.memex.system;
+package nu.marginalia.wmsa.memex.system.git;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
@@ -10,7 +10,8 @@ import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
-import org.eclipse.jgit.transport.*;
+import org.eclipse.jgit.transport.JschConfigSessionFactory;
+import org.eclipse.jgit.transport.SshSessionFactory;
 import org.eclipse.jgit.util.FS;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -19,13 +20,13 @@ import java.io.IOException;
 import java.nio.file.Path;
 
 @Singleton
-public class MemexGitRepo {
+public class MemexGitRepoImpl implements MemexGitRepo {
 
     private final Git git;
-    private final Logger logger = LoggerFactory.getLogger(MemexGitRepo.class);
+    private final Logger logger = LoggerFactory.getLogger(MemexGitRepoImpl.class);
 
     @Inject
-    public MemexGitRepo(@Named("memex-root") Path root) throws IOException {
+    public MemexGitRepoImpl(@Named("memex-root") Path root) throws IOException {
 
         FileRepositoryBuilder repositoryBuilder = new FileRepositoryBuilder();
 
@@ -49,6 +50,7 @@ public class MemexGitRepo {
         pull();
     }
 
+    @Override
     public void pull() {
         try {
             git.pull().call();
@@ -58,6 +60,7 @@ public class MemexGitRepo {
         }
     }
 
+    @Override
     public void remove(MemexNodeUrl url) {
         try {
             git.rm()
@@ -72,6 +75,7 @@ public class MemexGitRepo {
         }
     }
 
+    @Override
     public void add(MemexNodeUrl url) {
         try {
             git.add()
@@ -87,6 +91,7 @@ public class MemexGitRepo {
             logger.error("Git operation failed", ex);
         }
     }
+    @Override
     public void update(MemexNodeUrl url) {
         try {
             git.add()
@@ -105,6 +110,7 @@ public class MemexGitRepo {
     }
 
 
+    @Override
     public void rename(MemexNodeUrl src, MemexNodeUrl dst) {
         try {
             git.rm().addFilepattern(filePattern(src)).call();

marginalia_nu/src/test/java/nu/marginalia/wmsa/memex/change/GemtextChangeTest.java

@@ -2,16 +2,18 @@ package nu.marginalia.wmsa.memex.change;
 
 import io.reactivex.rxjava3.plugins.RxJavaPlugins;
 import lombok.SneakyThrows;
-import nu.marginalia.gemini.GeminiService;
+import nu.marginalia.gemini.GeminiServiceImpl;
 import nu.marginalia.util.test.TestUtil;
-import nu.marginalia.wmsa.memex.*;
+import nu.marginalia.wmsa.memex.Memex;
+import nu.marginalia.wmsa.memex.MemexData;
+import nu.marginalia.wmsa.memex.MemexLoader;
 import nu.marginalia.wmsa.memex.model.MemexNodeHeadingId;
 import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
 import nu.marginalia.wmsa.memex.renderer.MemexRendererers;
 import nu.marginalia.wmsa.memex.system.MemexFileSystemModifiedTimes;
 import nu.marginalia.wmsa.memex.system.MemexFileWriter;
-import nu.marginalia.wmsa.memex.system.MemexGitRepo;
 import nu.marginalia.wmsa.memex.system.MemexSourceFileSystem;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepoImpl;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -61,13 +63,13 @@ class GemtextChangeTest {
         var data = new MemexData();
 
         memex = new Memex(data, null,
-                Mockito.mock(MemexGitRepo.class), new MemexLoader(data, new MemexFileSystemModifiedTimes(),
+                Mockito.mock(MemexGitRepoImpl.class), new MemexLoader(data, new MemexFileSystemModifiedTimes(),
-                    new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepo.class)),
+                    new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepoImpl.class)),
                 tempDir, tombstonePath, redirectPath),
                 Mockito.mock(MemexFileWriter.class),
                 null,
                 Mockito.mock(MemexRendererers.class),
-                Mockito.mock(GeminiService.class));
+                Mockito.mock(GeminiServiceImpl.class));
     }
 
     @SneakyThrows

marginalia_nu/src/test/java/nu/marginalia/wmsa/memex/change/GemtextTaskUpdateTest.java

@@ -2,18 +2,20 @@ package nu.marginalia.wmsa.memex.change;
 
 import io.reactivex.rxjava3.plugins.RxJavaPlugins;
 import lombok.SneakyThrows;
-import nu.marginalia.gemini.GeminiService;
+import nu.marginalia.gemini.GeminiServiceImpl;
 import nu.marginalia.gemini.gmi.GemtextDocument;
 import nu.marginalia.util.test.TestUtil;
-import nu.marginalia.wmsa.memex.*;
+import nu.marginalia.wmsa.memex.Memex;
+import nu.marginalia.wmsa.memex.MemexData;
+import nu.marginalia.wmsa.memex.MemexLoader;
 import nu.marginalia.wmsa.memex.change.update.GemtextDocumentUpdateCalculator;
 import nu.marginalia.wmsa.memex.model.MemexNodeHeadingId;
 import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
 import nu.marginalia.wmsa.memex.renderer.MemexRendererers;
 import nu.marginalia.wmsa.memex.system.MemexFileSystemModifiedTimes;
 import nu.marginalia.wmsa.memex.system.MemexFileWriter;
-import nu.marginalia.wmsa.memex.system.MemexGitRepo;
 import nu.marginalia.wmsa.memex.system.MemexSourceFileSystem;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepoImpl;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -67,12 +69,12 @@ class GemtextTaskUpdateTest {
         Files.createDirectory(tempDir.resolve("special"));
         var data = new MemexData();
 
-        memex = new Memex(data, null, Mockito.mock(MemexGitRepo.class), new MemexLoader(data, new MemexFileSystemModifiedTimes(),
+        memex = new Memex(data, null, Mockito.mock(MemexGitRepoImpl.class), new MemexLoader(data, new MemexFileSystemModifiedTimes(),
-                new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepo.class)), tempDir, tombstonePath, redirectPath),
+                new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepoImpl.class)), tempDir, tombstonePath, redirectPath),
                 Mockito.mock(MemexFileWriter.class),
                 null,
                 Mockito.mock(MemexRendererers.class),
-                Mockito.mock(GeminiService.class));
+                Mockito.mock(GeminiServiceImpl.class));
     }
 
     @SneakyThrows

marginalia_nu/src/test/java/nu/marginalia/wmsa/memex/change/GemtextTombstoneUpdateCaclulatorTest.java

@@ -2,15 +2,17 @@ package nu.marginalia.wmsa.memex.change;
 
 import io.reactivex.rxjava3.plugins.RxJavaPlugins;
 import lombok.SneakyThrows;
-import nu.marginalia.gemini.GeminiService;
+import nu.marginalia.gemini.GeminiServiceImpl;
 import nu.marginalia.util.test.TestUtil;
-import nu.marginalia.wmsa.memex.*;
+import nu.marginalia.wmsa.memex.Memex;
+import nu.marginalia.wmsa.memex.MemexData;
+import nu.marginalia.wmsa.memex.MemexLoader;
 import nu.marginalia.wmsa.memex.model.MemexNodeUrl;
 import nu.marginalia.wmsa.memex.renderer.MemexRendererers;
 import nu.marginalia.wmsa.memex.system.MemexFileSystemModifiedTimes;
 import nu.marginalia.wmsa.memex.system.MemexFileWriter;
-import nu.marginalia.wmsa.memex.system.MemexGitRepo;
 import nu.marginalia.wmsa.memex.system.MemexSourceFileSystem;
+import nu.marginalia.wmsa.memex.system.git.MemexGitRepoImpl;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -64,13 +66,13 @@ class GemtextTombstoneUpdateCaclulatorTest {
         var data = new MemexData();
 
         memex = new Memex(data, null,
-                Mockito.mock(MemexGitRepo.class),
+                Mockito.mock(MemexGitRepoImpl.class),
                 new MemexLoader(data, new MemexFileSystemModifiedTimes(),
-                    new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepo.class)), tempDir, tombstonePath, redirectPath),
+                    new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepoImpl.class)), tempDir, tombstonePath, redirectPath),
                 Mockito.mock(MemexFileWriter.class),
                 updateCaclulator,
                 Mockito.mock(MemexRendererers.class),
-                Mockito.mock(GeminiService.class));
+                Mockito.mock(GeminiServiceImpl.class));
     }
 
     @SneakyThrows