Merge pull request #39 from MarginaliaSearch/master-control-program

Message Queue, State Machine, and Control Service

build.gradle

@@ -9,14 +9,37 @@ version 'SNAPSHOT'
 compileJava.options.encoding = "UTF-8"
 compileTestJava.options.encoding = "UTF-8"
 
-task dist(type: Copy) {
+tasks.register('dist', Copy) {
     from subprojects.collect { it.tasks.withType(Tar) }
     into "$buildDir/dist"
-}
 
+    doLast {
+        copy {
+            from tarTree("$buildDir/dist/converter-process.tar")
+            into "$projectDir/run/dist/"
+        }
+        copy {
+            from tarTree("$buildDir/dist/crawler-process.tar")
+            into "$projectDir/run/dist/"
+        }
+        copy {
+            from tarTree("$buildDir/dist/loader-process.tar")
+            into "$projectDir/run/dist/"
+        }
+        copy {
+            from tarTree("$buildDir/dist/website-adjacencies-calculator.tar")
+            into "$projectDir/run/dist/"
+        }
+        copy {
+            from tarTree("$buildDir/dist/crawl-job-extractor-process.tar")
+            into "$projectDir/run/dist/"
+        }
+    }
+}
 idea {
     module {
         excludeDirs.add(file("$projectDir/run/model"))
+        excludeDirs.add(file("$projectDir/run/dist"))
         excludeDirs.add(file("$projectDir/run/samples"))
         excludeDirs.add(file("$projectDir/run/db"))
         excludeDirs.add(file("$projectDir/run/logs"))

code/api/index-api/build.gradle

@@ -16,7 +16,7 @@ dependencies {
     implementation project(':code:common:config')
     implementation project(':code:common:service-discovery')
     implementation project(':code:common:service-client')
-
+    implementation project(':code:common:message-queue')
     implementation project(':code:features-index:index-query')
 
     implementation libs.lombok

code/api/index-api/src/main/java/nu/marginalia/index/client/IndexClient.java

@@ -8,27 +8,41 @@ import nu.marginalia.WmsaHome;
 import nu.marginalia.client.AbstractDynamicClient;
 import nu.marginalia.client.Context;
 import nu.marginalia.index.client.model.query.SearchSpecification;
-import nu.marginalia.index.client.model.results.SearchResultItem;
 import nu.marginalia.index.client.model.results.SearchResultSet;
 import nu.marginalia.model.gson.GsonFactory;
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.service.descriptor.ServiceDescriptors;
 import nu.marginalia.service.id.ServiceId;
 
 import javax.annotation.CheckReturnValue;
-import java.util.List;
+import java.util.UUID;
 
 @Singleton
 public class IndexClient extends AbstractDynamicClient {
 
     private static final Summary wmsa_search_index_api_time = Summary.build().name("wmsa_search_index_api_time").help("-").register();
 
+    private final MqOutbox outbox;
+
     @Inject
-    public IndexClient(ServiceDescriptors descriptors) {
+    public IndexClient(ServiceDescriptors descriptors,
+                       MessageQueueFactory messageQueueFactory) {
         super(descriptors.forId(ServiceId.Index), WmsaHome.getHostsFile(), GsonFactory::get);
 
+        String inboxName = ServiceId.Index.name + ":" + "0";
+        String outboxName = System.getProperty("service-name", UUID.randomUUID().toString());
+
+        outbox = messageQueueFactory.createOutbox(inboxName, outboxName, UUID.randomUUID());
+
         setTimeout(30);
     }
 
+
+    public MqOutbox outbox() {
+        return outbox;
+    }
+
     @CheckReturnValue
     public SearchResultSet query(Context ctx, SearchSpecification specs) {
         return wmsa_search_index_api_time.time(

code/api/index-api/src/main/java/nu/marginalia/index/client/IndexMqEndpoints.java

@@ -0,0 +1,10 @@
+package nu.marginalia.index.client;
+
+public class IndexMqEndpoints {
+    public static final String INDEX_IS_BLOCKED = "INDEX-IS-BLOCKED";
+    public static final String INDEX_REPARTITION = "INDEX-REPARTITION";
+
+    public static final String INDEX_RELOAD_LEXICON = "INDEX-RELOAD-LEXICON";
+    public static final String INDEX_REINDEX = "INDEX-REINDEX";
+
+}

code/api/index-api/src/main/java/nu/marginalia/index/client/model/query/SearchSetIdentifier.java

@@ -8,6 +8,7 @@ package nu.marginalia.index.client.model.query;
 public enum SearchSetIdentifier {
     NONE,
     RETRO,
+    BLOGS,
     ACADEMIA,
     SMALLWEB
 }

code/api/process-mqapi/build.gradle

@@ -0,0 +1,30 @@
+plugins {
+    id 'java'
+    id "io.freefair.lombok" version "5.3.3.3"
+
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(17))
+    }
+}
+
+dependencies {
+    implementation project(':code:common:db')
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+}
+
+test {
+    useJUnitPlatform()
+}
+
+task fastTests(type: Test) {
+    useJUnitPlatform {
+        excludeTags "slow"
+    }
+}

code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/ProcessInboxNames.java

@@ -0,0 +1,7 @@
+package nu.marginalia.mqapi;
+
+public class ProcessInboxNames {
+    public static final String CONVERTER_INBOX = "converter";
+    public static final String LOADER_INBOX = "loader";
+    public static final String CRAWLER_INBOX = "crawler";
+}

code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertAction.java

@@ -0,0 +1,7 @@
+package nu.marginalia.mqapi.converting;
+
+public enum ConvertAction {
+    ConvertCrawlData,
+    SideloadEncyclopedia,
+    SideloadStackexchange
+}

code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java

@@ -0,0 +1,12 @@
+package nu.marginalia.mqapi.converting;
+
+import lombok.AllArgsConstructor;
+import nu.marginalia.db.storage.model.FileStorageId;
+
+@AllArgsConstructor
+public class ConvertRequest {
+    public final ConvertAction action;
+    public final String inputSource;
+    public final FileStorageId crawlStorage;
+    public final FileStorageId processedDataStorage;
+}

code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/crawling/CrawlRequest.java

@@ -0,0 +1,11 @@
+package nu.marginalia.mqapi.crawling;
+
+import lombok.AllArgsConstructor;
+import nu.marginalia.db.storage.model.FileStorageId;
+
+/** A request to start a crawl */
+@AllArgsConstructor
+public class CrawlRequest {
+    public FileStorageId specStorage;
+    public FileStorageId crawlStorage;
+}

code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/loading/LoadRequest.java

@@ -0,0 +1,9 @@
+package nu.marginalia.mqapi.loading;
+
+import lombok.AllArgsConstructor;
+import nu.marginalia.db.storage.model.FileStorageId;
+
+@AllArgsConstructor
+public class LoadRequest {
+    public FileStorageId processedDataStorage;
+}

code/api/readme.md

@@ -1,4 +1,10 @@
-# Core Service Clients
+# Clients
+
+## Core Services
+
+* [assistant-api](assistant-api/)
+* [search-api](search-api/)
+* [index-api](index-api/)
 
 These are clients for the [core services](../services-core/), along with what models
 are necessary for speaking to them. They each implement the abstract client classes from
@@ -8,3 +14,10 @@ All that is necessary is to `@Inject` them into the constructor and then
 requests can be sent. 
 
 **Note:** If you are looking for the public API, it's handled by the api service in [services-satellite/api-service](../services-satellite/api-service).
+
+## MQ-API Process API
+
+[process-mqapi](process-mqapi/) defines requests and inboxes for the message queue based API used 
+for interacting with processes.   
+
+See [common/message-queue](../common/message-queue) and [services-satellite/control-service](../services-satellite/control-service). 

code/api/search-api/build.gradle

@@ -14,6 +14,7 @@ java {
 dependencies {
     implementation project(':code:common:model')
     implementation project(':code:common:config')
+    implementation project(':code:common:message-queue')
     implementation project(':code:common:service-discovery')
     implementation project(':code:common:service-client')
 

code/api/search-api/src/main/java/nu/marginalia/search/client/SearchClient.java

@@ -5,6 +5,8 @@ import com.google.inject.Singleton;
 import io.reactivex.rxjava3.core.Observable;
 import nu.marginalia.client.AbstractDynamicClient;
 import nu.marginalia.model.gson.GsonFactory;
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.search.client.model.ApiSearchResults;
 import nu.marginalia.service.descriptor.ServiceDescriptors;
 import nu.marginalia.service.id.ServiceId;
@@ -16,14 +18,30 @@ import org.slf4j.LoggerFactory;
 import javax.annotation.CheckReturnValue;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.util.UUID;
 
 @Singleton
 public class SearchClient extends AbstractDynamicClient {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
+    private final MqOutbox outbox;
+
     @Inject
-    public SearchClient(ServiceDescriptors descriptors) {
+    public SearchClient(ServiceDescriptors descriptors,
+                        MessageQueueFactory messageQueueFactory) {
+
         super(descriptors.forId(ServiceId.Search), WmsaHome.getHostsFile(), GsonFactory::get);
+
+        String inboxName = ServiceId.Search.name + ":" + "0";
+        String outboxName = System.getProperty("service-name", UUID.randomUUID().toString());
+
+        outbox = messageQueueFactory.createOutbox(inboxName, outboxName, UUID.randomUUID());
+
+    }
+
+
+    public MqOutbox outbox() {
+        return outbox;
     }
 
     @CheckReturnValue

code/api/search-api/src/main/java/nu/marginalia/search/client/SearchMqEndpoints.java

@@ -0,0 +1,6 @@
+package nu.marginalia.search.client;
+
+public class SearchMqEndpoints {
+    /** Flushes the URL caches, run if significant changes have occurred in the URLs database */
+    public static final String FLUSH_CACHES = "FLUSH_CACHES";
+}

code/common/config/src/main/java/nu/marginalia/WmsaHome.java

@@ -10,7 +10,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Optional;
-import java.util.Properties;
 import java.util.stream.Stream;
 
 public class WmsaHome {
@@ -79,35 +78,6 @@ public class WmsaHome {
         return getHomePath().resolve("data").resolve("IP2LOCATION-LITE-DB1.CSV");
     }
 
-    public static Path getDisk(String name) {
-        var pathStr = getDiskProperties().getProperty(name);
-        if (null == pathStr) {
-            throw new RuntimeException("Disk " + name + " was not configured");
-        }
-        Path p = Path.of(pathStr);
-        if (!Files.isDirectory(p)) {
-            throw new RuntimeException("Disk " + name + " does not exist or is not a directory!");
-        }
-        return p;
-    }
-
-    public static Properties getDiskProperties() {
-        Path settingsFile = getHomePath().resolve("conf/disks.properties");
-
-        if (!Files.isRegularFile(settingsFile)) {
-            throw new RuntimeException("Could not find disk settings " + settingsFile);
-        }
-
-        try (var is = Files.newInputStream(settingsFile)) {
-            var props = new Properties();
-            props.load(is);
-            return props;
-        }
-        catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
     public static LanguageModels getLanguageModels() {
         final Path home = getHomePath();
 

code/common/db/build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'java'
     id "io.freefair.lombok" version "5.3.3.3"
     id 'jvm-test-suite'
-
+    id "org.flywaydb.flyway" version "8.2.0"
 }
 
 java {
@@ -11,6 +11,10 @@ java {
     }
 }
 
+configurations {
+    flywayMigration.extendsFrom(implementation)
+}
+
 dependencies {
     implementation project(':code:common:model')
 
@@ -29,6 +33,7 @@ dependencies {
 
     implementation libs.rxjava
     implementation libs.bundles.mariadb
+    flywayMigration 'org.flywaydb:flyway-mysql:9.8.1'
 
     testImplementation libs.bundles.slf4j.test
     testImplementation libs.bundles.junit
@@ -40,6 +45,15 @@ dependencies {
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
 }
 
+flyway {
+    url = 'jdbc:mariadb://localhost:3306/WMSA_prod'
+    user = 'wmsa'
+    password = 'wmsa'
+    schemas = ['WMSA_prod']
+    configurations = [ 'compileClasspath', 'flywayMigration' ]
+    locations = ['filesystem:src/main/resources/db/migration']
+}
+
 
 test {
     maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1

code/common/db/readme.md

@@ -2,10 +2,29 @@
 
 This module primarily contains SQL files for the URLs database. The most central tables are `EC_DOMAIN`, `EC_URL` and `EC_PAGE_DATA`.
 
+## Flyway
+
+The system uses flyway to track database changes and allow easy migrations, this is accessible via gradle tasks.
+
+* `flywayMigrate`
+* `flywayBaseline`
+* `flywayRepair`
+* `flywayClean` (dangerous as in wipes your entire database)
+
+Refer to the [Flyway documentation](https://documentation.red-gate.com/fd/flyway-documentation-138346877.html) for guidance.
+It's well documented and these are probably the only four tasks you'll ever need.
+
+If you are not running the system via docker, you need to provide alternative connection details than
+the defaults (TODO: how?).
+
+The migration files are in [resources/db/migration](src/main/resources/db/migration).  The file name convention
+incorporates the project's cal-ver versioning; and are applied in lexicographical order.
+
+    VYY_MM_v_nnn__description.sql
+
 ## Central Paths
 
-* [current](src/main/resources/sql/current) - The current database model
-* [migrations](src/main/resources/sql/migrations)
+* [migrations](src/main/resources/db/migration) - Flyway migrations
 
 ## See Also 
 

code/common/db/src/main/java/nu/marginalia/db/DomainBlacklistImpl.java

@@ -52,7 +52,7 @@ public class DomainBlacklistImpl implements DomainBlacklist {
         }
 
         try (var connection = dataSource.getConnection()) {
-            try (var stmt = connection.prepareStatement("SELECT EC_DOMAIN.ID FROM EC_DOMAIN INNER JOIN EC_DOMAIN_BLACKLIST ON EC_DOMAIN_BLACKLIST.URL_DOMAIN = EC_DOMAIN.DOMAIN_TOP")) {
+            try (var stmt = connection.prepareStatement("SELECT EC_DOMAIN.ID FROM EC_DOMAIN INNER JOIN EC_DOMAIN_BLACKLIST ON (EC_DOMAIN_BLACKLIST.URL_DOMAIN = EC_DOMAIN.DOMAIN_TOP OR EC_DOMAIN_BLACKLIST.URL_DOMAIN = EC_DOMAIN.DOMAIN_NAME)")) {
                 stmt.setFetchSize(1000);
                 var rsp = stmt.executeQuery();
                 while (rsp.next()) {

code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageManifest.java

@@ -0,0 +1,51 @@
+package nu.marginalia.db.storage;
+
+import com.google.gson.Gson;
+import nu.marginalia.db.storage.model.FileStorage;
+import nu.marginalia.db.storage.model.FileStorageType;
+import nu.marginalia.model.gson.GsonFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Optional;
+
+record FileStorageManifest(FileStorageType type, String description) {
+    private static final Gson gson = GsonFactory.get();
+    private static final String fileName = "marginalia-manifest.json";
+    private static final Logger logger = LoggerFactory.getLogger(FileStorageManifest.class);
+
+    public static Optional<FileStorageManifest> find(Path directory) {
+        Path expectedFileName = directory.resolve(fileName);
+
+        if (!Files.isRegularFile(expectedFileName) ||
+            !Files.isReadable(expectedFileName)) {
+            return Optional.empty();
+        }
+
+        try (var reader = Files.newBufferedReader(expectedFileName)) {
+            return Optional.of(gson.fromJson(reader, FileStorageManifest.class));
+        }
+        catch (Exception e) {
+            logger.warn("Failed to read manifest " + expectedFileName, e);
+            return Optional.empty();
+        }
+    }
+
+    public void write(FileStorage dir) {
+        Path expectedFileName = dir.asPath().resolve(fileName);
+
+        try (var writer = Files.newBufferedWriter(expectedFileName,
+                StandardOpenOption.CREATE,
+                StandardOpenOption.TRUNCATE_EXISTING))
+        {
+            gson.toJson(this, writer);
+        }
+        catch (Exception e) {
+            logger.warn("Failed to write manifest " + expectedFileName, e);
+        }
+    }
+
+}

code/common/db/src/main/java/nu/marginalia/db/storage/FileStorageService.java

@@ -0,0 +1,432 @@
+package nu.marginalia.db.storage;
+
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.db.storage.model.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.sql.SQLException;
+import java.util.*;
+
+/** Manages file storage for processes and services
+ */
+@Singleton
+public class FileStorageService {
+    private final HikariDataSource dataSource;
+    private final Logger logger = LoggerFactory.getLogger(FileStorageService.class);
+    @Inject
+    public FileStorageService(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    public Optional<FileStorage> findFileStorageToDelete() {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                SELECT ID FROM FILE_STORAGE WHERE DO_PURGE LIMIT 1
+                """)) {
+            var rs = stmt.executeQuery();
+            if (rs.next()) {
+                return Optional.of(getStorage(new FileStorageId(rs.getLong(1))));
+            }
+        } catch (SQLException e) {
+            return Optional.empty();
+        }
+        return Optional.empty();
+    }
+
+    /** @return the storage base with the given id, or null if it does not exist */
+    public FileStorageBase getStorageBase(FileStorageBaseId type) throws SQLException  {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT ID, NAME, PATH, TYPE, PERMIT_TEMP
+                     FROM FILE_STORAGE_BASE WHERE ID = ?
+                     """)) {
+            stmt.setLong(1, type.id());
+            try (var rs = stmt.executeQuery()) {
+                if (rs.next()) {
+                    return new FileStorageBase(
+                            new FileStorageBaseId(rs.getLong(1)),
+                            FileStorageBaseType.valueOf(rs.getString(4)),
+                            rs.getString(2),
+                            rs.getString(3),
+                            rs.getBoolean(5)
+                    );
+                }
+            }
+        }
+        return null;
+    }
+
+    public void synchronizeStorageManifests(FileStorageBase base) {
+        Set<String> ignoredPaths = new HashSet<>();
+
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                SELECT PATH FROM FILE_STORAGE WHERE BASE_ID = ?
+                """)) {
+            stmt.setLong(1, base.id().id());
+            var rs = stmt.executeQuery();
+            while (rs.next()) {
+                ignoredPaths.add(rs.getString(1));
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+
+        File basePathFile = Path.of(base.path()).toFile();
+        File[] files = basePathFile.listFiles(pathname -> pathname.isDirectory() && !ignoredPaths.contains(pathname.getName()));
+        if (files == null) return;
+        for (File file : files) {
+            var maybeManifest = FileStorageManifest.find(file.toPath());
+            if (maybeManifest.isEmpty()) continue;
+            var manifest = maybeManifest.get();
+
+            logger.info("Discovered new file storage: " + file.getName() + " (" + manifest.type() + ")");
+
+            try (var conn = dataSource.getConnection();
+                 var stmt = conn.prepareStatement("""
+                    INSERT INTO FILE_STORAGE(BASE_ID, PATH, TYPE, DESCRIPTION)
+                    VALUES (?, ?, ?, ?)
+                    """)) {
+                stmt.setLong(1, base.id().id());
+                stmt.setString(2, file.getName());
+                stmt.setString(3, manifest.type().name());
+                stmt.setString(4, manifest.description());
+                stmt.execute();
+                conn.commit();
+
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+    public void relateFileStorages(FileStorageId source, FileStorageId target) {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                INSERT INTO FILE_STORAGE_RELATION(SOURCE_ID, TARGET_ID) VALUES (?, ?)
+                """)) {
+            stmt.setLong(1, source.id());
+            stmt.setLong(2, target.id());
+            stmt.executeUpdate();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public List<FileStorage> getSourceFromStorage(FileStorage storage) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT SOURCE_ID FROM FILE_STORAGE_RELATION WHERE TARGET_ID = ?
+                     """)) {
+            stmt.setLong(1, storage.id().id());
+            var rs = stmt.executeQuery();
+            List<FileStorage> ret = new ArrayList<>();
+            while (rs.next()) {
+                ret.add(getStorage(new FileStorageId(rs.getLong(1))));
+            }
+            return ret;
+        }
+    }
+
+    public List<FileStorage> getTargetFromStorage(FileStorage storage) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT TARGET_ID FROM FILE_STORAGE_RELATION WHERE SOURCE_ID = ?
+                     """)) {
+            stmt.setLong(1, storage.id().id());
+            var rs = stmt.executeQuery();
+            List<FileStorage> ret = new ArrayList<>();
+            while (rs.next()) {
+                ret.add(getStorage(new FileStorageId(rs.getLong(1))));
+            }
+            return ret;
+        }
+    }
+
+    /** @return the storage base with the given type, or null if it does not exist */
+    public FileStorageBase getStorageBase(FileStorageBaseType type) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT ID, NAME, PATH, TYPE, PERMIT_TEMP
+                     FROM FILE_STORAGE_BASE WHERE TYPE = ?
+                     """)) {
+            stmt.setString(1, type.name());
+            try (var rs = stmt.executeQuery()) {
+                if (rs.next()) {
+                    return new FileStorageBase(
+                            new FileStorageBaseId(rs.getLong(1)),
+                            FileStorageBaseType.valueOf(rs.getString(4)),
+                            rs.getString(2),
+                            rs.getString(3),
+                            rs.getBoolean(5)
+                    );
+                }
+            }
+        }
+        return null;
+    }
+
+    public FileStorageBase createStorageBase(String name, Path path, FileStorageBaseType type, boolean permitTemp) throws SQLException, FileNotFoundException {
+
+        if (!Files.exists(path)) {
+            throw new FileNotFoundException("Storage base path does not exist: " + path);
+        }
+
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     INSERT INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, PERMIT_TEMP)
+                     VALUES (?, ?, ?, ?)
+                     """)) {
+            stmt.setString(1, name);
+            stmt.setString(2, path.toString());
+            stmt.setString(3, type.name());
+            stmt.setBoolean(4, permitTemp);
+
+            int update = stmt.executeUpdate();
+            if (update < 0) {
+                throw new SQLException("Failed to create storage base");
+            }
+        }
+
+        return getStorageBase(type);
+    }
+
+    /** Allocate a temporary storage of the given type if temporary allocation is permitted */
+    public FileStorage allocateTemporaryStorage(FileStorageBase base,
+                                                FileStorageType type,
+                                                String prefix,
+                                                String description) throws IOException, SQLException
+    {
+        if (!base.permitTemp()) {
+            throw new IllegalArgumentException("Temporary storage not permitted in base "  + base.name());
+        }
+
+        Path tempDir = Files.createTempDirectory(base.asPath(), prefix,
+                PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x"))
+        );
+
+        String relDir = base.asPath().relativize(tempDir).normalize().toString();
+
+        try (var conn = dataSource.getConnection();
+             var insert = conn.prepareStatement("""
+                INSERT INTO FILE_STORAGE(PATH, TYPE, DESCRIPTION, BASE_ID)
+                VALUES (?, ?, ?, ?)
+                """);
+             var query = conn.prepareStatement("""
+                SELECT ID FROM FILE_STORAGE WHERE PATH = ? AND BASE_ID = ?
+                """)
+             ) {
+            insert.setString(1, relDir);
+            insert.setString(2, type.name());
+            insert.setString(3, description);
+            insert.setLong(4, base.id().id());
+
+            if (insert.executeUpdate() < 1) {
+                throw new SQLException("Failed to insert storage");
+            }
+
+
+            query.setString(1, relDir);
+            query.setLong(2, base.id().id());
+            var rs = query.executeQuery();
+
+            if (rs.next()) {
+                var storage = getStorage(new FileStorageId(rs.getLong("ID")));
+
+                // Write a manifest file so we can pick this up later without needing to insert it into DB
+                // (e.g. when loading from outside the system)
+                var manifest = new FileStorageManifest(type, description);
+                manifest.write(storage);
+
+                return storage;
+            }
+
+        }
+
+        throw new SQLException("Failed to insert storage");
+    }
+
+
+    /** Allocate permanent storage in base */
+    public FileStorage allocatePermanentStorage(FileStorageBase base, String relativePath, FileStorageType type, String description) throws IOException, SQLException {
+
+        Path newDir = base.asPath().resolve(relativePath);
+
+        if (Files.exists(newDir)) {
+            throw new IllegalArgumentException("Storage already exists: " + newDir);
+        }
+
+        Files.createDirectory(newDir, PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x")));
+
+        try (var conn = dataSource.getConnection();
+             var update = conn.prepareStatement("""
+                INSERT INTO FILE_STORAGE(PATH, TYPE, DESCRIPTION, BASE_ID)
+                VALUES (?, ?, ?, ?)
+                """);
+             var query = conn.prepareStatement("""
+                SELECT ID
+                FROM FILE_STORAGE WHERE PATH = ? AND BASE_ID = ?
+                """)
+        ) {
+            update.setString(1, relativePath);
+            update.setString(2, type.name());
+            update.setString(3, description);
+            update.setLong(4, base.id().id());
+
+            if (update.executeUpdate() < 1)
+                throw new SQLException("Failed to insert storage");
+
+            query.setString(1, relativePath);
+            query.setLong(2, base.id().id());
+            var rs = query.executeQuery();
+
+            if (rs.next()) {
+                return new FileStorage(
+                        new FileStorageId(rs.getLong("ID")),
+                        base,
+                        type,
+                        newDir.toString(),
+                        description
+                );
+            }
+
+        }
+
+        throw new SQLException("Failed to insert storage");
+    }
+
+    public FileStorage getStorageByType(FileStorageType type) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT PATH, DESCRIPTION, ID, BASE_ID
+                     FROM FILE_STORAGE_VIEW WHERE TYPE = ?
+                     """)) {
+            stmt.setString(1, type.name());
+
+            long storageId;
+            long baseId;
+            String path;
+            String description;
+
+            try (var rs = stmt.executeQuery()) {
+                if (rs.next()) {
+                    baseId = rs.getLong("BASE_ID");
+                    storageId = rs.getLong("ID");
+                    path = rs.getString("PATH");
+                    description = rs.getString("DESCRIPTION");
+                }
+                else {
+                    return null;
+                }
+
+                var base = getStorageBase(new FileStorageBaseId(baseId));
+
+                return new FileStorage(
+                        new FileStorageId(storageId),
+                        base,
+                        type,
+                        path,
+                        description
+                );
+            }
+        }
+    }
+
+    /** @return the storage with the given id, or null if it does not exist */
+    public FileStorage getStorage(FileStorageId id) throws SQLException {
+
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT PATH, TYPE, DESCRIPTION, ID, BASE_ID
+                     FROM FILE_STORAGE_VIEW WHERE ID = ?
+                     """)) {
+            stmt.setLong(1, id.id());
+
+            long storageId;
+            long baseId;
+            String path;
+            String description;
+            FileStorageType type;
+
+            try (var rs = stmt.executeQuery()) {
+                if (rs.next()) {
+                    baseId = rs.getLong("BASE_ID");
+                    storageId = rs.getLong("ID");
+                    type = FileStorageType.valueOf(rs.getString("TYPE"));
+                    path = rs.getString("PATH");
+                    description = rs.getString("DESCRIPTION");
+                }
+                else {
+                    return null;
+                }
+
+                var base = getStorageBase(new FileStorageBaseId(baseId));
+
+                return new FileStorage(
+                        new FileStorageId(storageId),
+                        base,
+                        type,
+                        path,
+                        description
+                );
+            }
+        }
+    }
+
+    public void removeFileStorage(FileStorageId id) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     DELETE FROM FILE_STORAGE WHERE ID = ?
+                     """)) {
+            stmt.setLong(1, id.id());
+            stmt.executeUpdate();
+        }
+    }
+
+    public List<FileStorage> getEachFileStorage() {
+        List<FileStorage> ret = new ArrayList<>();
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT PATH, TYPE, DESCRIPTION, ID, BASE_ID
+                     FROM FILE_STORAGE_VIEW
+                     """)) {
+
+            long storageId;
+            long baseId;
+            String path;
+            String description;
+            FileStorageType type;
+
+            try (var rs = stmt.executeQuery()) {
+                while (rs.next()) {
+                    baseId = rs.getLong("BASE_ID");
+                    storageId = rs.getLong("ID");
+                    path = rs.getString("PATH");
+                    type = FileStorageType.valueOf(rs.getString("TYPE"));
+                    description = rs.getString("DESCRIPTION");
+
+                    var base = getStorageBase(new FileStorageBaseId(baseId));
+
+                    ret.add(new FileStorage(
+                            new FileStorageId(storageId),
+                            base,
+                            type,
+                            path,
+                            description
+                    ));
+                }
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+
+        return ret;
+    }
+}

code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorage.java

@@ -0,0 +1,24 @@
+package nu.marginalia.db.storage.model;
+
+import java.nio.file.Path;
+
+/**
+ * Represents a file storage area
+ *
+ * @param id  the id of the storage in the database
+ * @param base the base of the storage
+ * @param type the type of data expected
+ * @param path the full path of the storage on disk
+ * @param description a description of the storage
+ */
+public record FileStorage(
+        FileStorageId id,
+        FileStorageBase base,
+        FileStorageType type,
+        String path,
+        String description)
+{
+    public Path asPath() {
+        return Path.of(path);
+    }
+}

code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBase.java

@@ -0,0 +1,23 @@
+package nu.marginalia.db.storage.model;
+
+import java.nio.file.Path;
+
+/**
+ * Represents a file storage base directory
+ *
+ * @param id  the id of the storage base in the database
+ * @param type  the type of the storage base
+ * @param name  the name of the storage base
+ * @param path  the path of the storage base
+ * @param permitTemp if true, the storage may be used for temporary files
+ */
+public record FileStorageBase(FileStorageBaseId id,
+                              FileStorageBaseType type,
+                              String name,
+                              String path,
+                              boolean permitTemp
+                              ) {
+    public Path asPath() {
+        return Path.of(path);
+    }
+}

code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseId.java

@@ -0,0 +1,8 @@
+package nu.marginalia.db.storage.model;
+
+public record FileStorageBaseId(long id) {
+
+    public String toString() {
+        return Long.toString(id);
+    }
+}

code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageBaseType.java

@@ -0,0 +1,8 @@
+package nu.marginalia.db.storage.model;
+
+public enum FileStorageBaseType {
+    SSD_INDEX,
+    SSD_WORK,
+    SLOW,
+    BACKUP
+}

code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageId.java

@@ -0,0 +1,14 @@
+package nu.marginalia.db.storage.model;
+
+public record FileStorageId(long id) {
+    public static FileStorageId parse(String str) {
+        return new FileStorageId(Long.parseLong(str));
+    }
+    public static FileStorageId of(int storageId) {
+        return new FileStorageId(storageId);
+    }
+
+    public String toString() {
+        return Long.toString(id);
+    }
+}

code/common/db/src/main/java/nu/marginalia/db/storage/model/FileStorageType.java

@@ -0,0 +1,14 @@
+package nu.marginalia.db.storage.model;
+
+public enum FileStorageType {
+    CRAWL_SPEC,
+    CRAWL_DATA,
+    PROCESSED_DATA,
+    INDEX_STAGING,
+    LEXICON_STAGING,
+    INDEX_LIVE,
+    LEXICON_LIVE,
+    BACKUP,
+    EXPORT,
+    SEARCH_SETS
+}

code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql

@@ -2,6 +2,7 @@
 CREATE TABLE IF NOT EXISTS EC_DOMAIN_BLACKLIST (
     ID INT PRIMARY KEY AUTO_INCREMENT,
     URL_DOMAIN VARCHAR(255) UNIQUE NOT NULL
+    COMMENT VARCHAR(255) DEFAULT NULL
 )
 CHARACTER SET utf8mb4
 COLLATE utf8mb4_unicode_ci;

code/common/db/src/main/resources/db/migration/V23_07_0_002__service_status.sql

@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS SERVICE_HEARTBEAT (
+    SERVICE_NAME VARCHAR(255) PRIMARY KEY COMMENT "Full name of the service, including node id if applicable, e.g. search-service:0",
+    SERVICE_BASE VARCHAR(255) NOT NULL COMMENT "Base name of the service, e.g. search-service",
+    INSTANCE VARCHAR(255) NOT NULL COMMENT "UUID of the service instance",
+    ALIVE BOOLEAN NOT NULL DEFAULT TRUE COMMENT "Set to false when the service is doing an orderly shutdown",
+    HEARTBEAT_TIME TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT "Service was last seen at this point"
+);
+
+CREATE TABLE IF NOT EXISTS PROCESS_HEARTBEAT (
+    PROCESS_NAME VARCHAR(255) PRIMARY KEY COMMENT "Full name of the process, including node id if applicable, e.g. converter:0",
+    PROCESS_BASE VARCHAR(255) NOT NULL COMMENT "Base name of the process, e.g. converter",
+    INSTANCE VARCHAR(255) NOT NULL COMMENT "UUID of the process instance",
+    STATUS ENUM ('STARTING', 'RUNNING', 'STOPPED') NOT NULL DEFAULT 'STARTING' COMMENT "Status of the process",
+    PROGRESS INT NOT NULL DEFAULT 0 COMMENT "Progress of the process",
+    HEARTBEAT_TIME TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT "Process was last seen at this point"
+);
+
+CREATE TABLE IF NOT EXISTS SERVICE_EVENTLOG(
+    ID BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT "Unique id",
+    SERVICE_NAME VARCHAR(255) NOT NULL COMMENT "Full name of the service, including node id if applicable, e.g. search-service:0",
+    SERVICE_BASE VARCHAR(255) NOT NULL COMMENT "Base name of the service, e.g. search-service",
+    INSTANCE VARCHAR(255) NOT NULL COMMENT "UUID of the service instance",
+    EVENT_TIME TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT "Event time",
+    EVENT_TYPE VARCHAR(255) NOT NULL COMMENT "Event type",
+    EVENT_MESSAGE VARCHAR(255) NOT NULL COMMENT "Event message"
+);
+

code/common/db/src/main/resources/db/migration/V23_07_0_003__message_queue.sql

@@ -0,0 +1,21 @@
+CREATE TABLE IF NOT EXISTS MESSAGE_QUEUE (
+    ID              BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT 'Unique id',
+    RELATED_ID      BIGINT NOT NULL DEFAULT -1        COMMENT 'Unique id a related message',
+    SENDER_INBOX    VARCHAR(255)          COMMENT 'Name of the sender inbox',
+    RECIPIENT_INBOX VARCHAR(255) NOT NULL COMMENT 'Name of the recipient inbox',
+    FUNCTION        VARCHAR(255) NOT NULL COMMENT 'Which function to run',
+    PAYLOAD         TEXT                  COMMENT 'Message to recipient',
+    -- These fields are used to avoid double processing of messages
+    -- instance marks the unique instance of the party, and the tick marks
+    -- the current polling iteration.  Both are necessary.
+    OWNER_INSTANCE  VARCHAR(255)          COMMENT 'Instance UUID corresponding to the party that has claimed the message',
+    OWNER_TICK      BIGINT  DEFAULT -1    COMMENT 'Used by recipient to determine which messages it has processed',
+    STATE           ENUM('NEW', 'ACK', 'OK', 'ERR', 'DEAD')
+                    NOT NULL DEFAULT 'NEW' COMMENT 'Processing state',
+    CREATED_TIME    TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT 'Time of creation',
+    UPDATED_TIME    TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT 'Time of last update',
+    TTL             INT              COMMENT 'Time to live in seconds'
+);
+
+CREATE INDEX MESSAGE_QUEUE_STATE_IDX ON MESSAGE_QUEUE(STATE);
+CREATE INDEX MESSAGE_QUEUE_OI_TICK_IDX ON MESSAGE_QUEUE(OWNER_INSTANCE, OWNER_TICK);

code/common/db/src/main/resources/db/migration/V23_07_0_004__file_storage.sql

@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS FILE_STORAGE_BASE (
+    ID BIGINT PRIMARY KEY AUTO_INCREMENT,
+    NAME VARCHAR(255) NOT NULL UNIQUE,
+    PATH VARCHAR(255) NOT NULL UNIQUE COMMENT 'The path to the storage base',
+    TYPE ENUM ('SSD_INDEX', 'SSD_WORK', 'SLOW', 'BACKUP') NOT NULL,
+    PERMIT_TEMP BOOLEAN NOT NULL DEFAULT FALSE COMMENT 'If true, the storage can be used for temporary files'
+)
+CHARACTER SET utf8mb4
+COLLATE utf8mb4_bin;
+
+CREATE TABLE IF NOT EXISTS FILE_STORAGE (
+    ID BIGINT PRIMARY KEY AUTO_INCREMENT,
+    BASE_ID BIGINT NOT NULL,
+    PATH VARCHAR(255) NOT NULL COMMENT 'The path to the storage relative to the base',
+    DESCRIPTION VARCHAR(255) NOT NULL,
+    TYPE ENUM ('CRAWL_SPEC', 'CRAWL_DATA', 'PROCESSED_DATA', 'INDEX_STAGING', 'LEXICON_STAGING', 'INDEX_LIVE', 'LEXICON_LIVE', 'SEARCH_SETS', 'BACKUP', 'EXPORT') NOT NULL,
+    DO_PURGE BOOLEAN NOT NULL DEFAULT FALSE COMMENT 'If true, the storage may be cleaned',
+    CREATE_DATE TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+    CONSTRAINT CONS UNIQUE (BASE_ID, PATH),
+    FOREIGN KEY (BASE_ID) REFERENCES FILE_STORAGE_BASE(ID) ON DELETE CASCADE
+)
+CHARACTER SET utf8mb4
+COLLATE utf8mb4_bin;
+
+CREATE TABLE IF NOT EXISTS FILE_STORAGE_RELATION (
+    SOURCE_ID BIGINT NOT NULL,
+    TARGET_ID BIGINT NOT NULL,
+    CONSTRAINT CONS UNIQUE (SOURCE_ID, TARGET_ID),
+    FOREIGN KEY (SOURCE_ID) REFERENCES FILE_STORAGE(ID) ON DELETE CASCADE,
+    FOREIGN KEY (TARGET_ID) REFERENCES FILE_STORAGE(ID) ON DELETE CASCADE
+);
+
+CREATE VIEW FILE_STORAGE_VIEW
+AS SELECT
+    CONCAT(BASE.PATH, '/', STORAGE.PATH) AS PATH,
+    STORAGE.TYPE AS TYPE,
+    DESCRIPTION AS DESCRIPTION,
+    CREATE_DATE AS CREATE_DATE,
+    STORAGE.ID AS ID,
+    BASE.ID AS BASE_ID
+FROM FILE_STORAGE STORAGE
+INNER JOIN FILE_STORAGE_BASE BASE ON STORAGE.BASE_ID=BASE.ID;

code/common/db/src/main/resources/db/migration/V23_07_0_005__file_storage_default_values.sql

@@ -0,0 +1,28 @@
+INSERT IGNORE INTO FILE_STORAGE_BASE(NAME, PATH, TYPE, PERMIT_TEMP)
+VALUES
+('Index Storage', '/vol', 'SSD_INDEX', false),
+('Data Storage', '/samples', 'SLOW', true);
+
+INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE)
+SELECT ID, 'iw', "Index Staging Area", 'INDEX_STAGING'
+FROM FILE_STORAGE_BASE WHERE NAME='Index Storage';
+
+INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE)
+SELECT ID, 'ir', "Index Live Area", 'INDEX_LIVE'
+FROM FILE_STORAGE_BASE WHERE NAME='Index Storage';
+
+INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE)
+SELECT ID, 'lw', "Lexicon Staging Area", 'LEXICON_STAGING'
+FROM FILE_STORAGE_BASE WHERE NAME='Index Storage';
+
+INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE)
+SELECT ID, 'lr', "Lexicon Live Area", 'LEXICON_LIVE'
+FROM FILE_STORAGE_BASE WHERE NAME='Index Storage';
+
+INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE)
+SELECT ID, 'ss', "Search Sets", 'SEARCH_SETS'
+FROM FILE_STORAGE_BASE WHERE NAME='Index Storage';
+
+INSERT IGNORE INTO FILE_STORAGE(BASE_ID, PATH, DESCRIPTION, TYPE)
+SELECT ID, 'export', "Exported Data", 'EXPORT'
+FROM FILE_STORAGE_BASE WHERE TYPE='EXPORT';

code/common/db/src/main/resources/db/migration/V23_07_0_006__message_queue_default_jobs.sql

@@ -0,0 +1,7 @@
+INSERT INTO MESSAGE_QUEUE(RECIPIENT_INBOX,FUNCTION,PAYLOAD) VALUES
+	 ('fsm:converter_monitor','INITIAL',''),
+	 ('fsm:loader_monitor','INITIAL',''),
+	 ('fsm:crawler_monitor','INITIAL',''),
+	 ('fsm:message_queue_monitor','INITIAL',''),
+	 ('fsm:process_liveness_monitor','INITIAL',''),
+	 ('fsm:file_storage_monitor','INITIAL','');

code/common/db/src/main/resources/db/migration/V23_07_0_007__task_status.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS TASK_HEARTBEAT (
+    TASK_NAME VARCHAR(255) PRIMARY KEY COMMENT "Full name of the task, including node id if applicable, e.g. reconvert:0",
+    TASK_BASE VARCHAR(255) NOT NULL COMMENT "Base name of the task, e.g. reconvert",
+    INSTANCE VARCHAR(255) NOT NULL COMMENT "UUID of the task instance",
+    SERVICE_INSTANCE VARCHAR(255) NOT NULL COMMENT "UUID of the parent service",
+    STATUS ENUM ('STARTING', 'RUNNING', 'STOPPED') NOT NULL DEFAULT 'STARTING' COMMENT "Status of the task",
+    PROGRESS INT NOT NULL DEFAULT 0 COMMENT "Progress of the task",
+    STAGE_NAME VARCHAR(255) DEFAULT "",
+    HEARTBEAT_TIME TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT "Task was last seen at this point"
+);

code/common/db/src/main/resources/sql/migrations/00-news-items.sql

@@ -1,76 +0,0 @@
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'A search engine that favors text-heavy sites and punishes modern web design',
-'https://news.ycombinator.com/item?id=28550764',
-'Hacker News',
-'2021-09-16'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'A Search Engine Designed To Surprise You',
-'https://onezero.medium.com/a-search-engine-designed-to-surprise-you-b81944ed5c06',
-'Clive Thompson OneZero',
-'2021-09-16'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'🎂 First anniversary! 🎊',
-'https://memex.marginalia.nu/log/49-marginalia-1-year.gmi',
-null,
-'2022-02-26');
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'Marginalia Search - Serendipity Engineering',
-'https://www.metafilter.com/194653/Marginalia-Search-Serendipity-Engineering',
-'MetaFilter',
-'2022-03-09');
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'What Google Search Isn\'t Showing You',
-'https://www.newyorker.com/culture/infinite-scroll/what-google-search-isnt-showing-you',
-'The New Yorker 🎩',
-'2022-03-10'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'You Should Check Out the Indie Web 🎞️',
-'https://www.youtube.com/watch?v=rTSEr0cRJY8',
-'YouTube, You\'ve Got Kat',
-'2022-03-15'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'Marginalia Goes Open Source',
-'https://news.ycombinator.com/item?id=31536626',
-'Hacker News',
-'2022-05-28'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'Kritik an Googles Suche - Platzhirsch auf dem Nebenschauplatz',
-'https://www.deutschlandfunkkultur.de/google-suche-100.html',
-'Deutschlandfunk Kultur 🇩🇪',
-'2022-08-18'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'Google ei enää tideä',
-'https://www.hs.fi/visio/art-2000009139237.html',
-'Helsing Sanomat 🇫🇮',
-'2022-10-19'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'Marginalia\'s Index Reaches 100,000,000 Documents 🎊',
-'https://memex.marginalia.nu/log/64-hundred-million.gmi',
-null,
-'2022-10-21'
-);
-
-INSERT IGNORE INTO SEARCH_NEWS_FEED(TITLE, LINK, SOURCE, LIST_DATE) VALUES (
-'Marginalia Receives NLnet grant',
-'https://memex.marginalia.nu/log/74-marginalia-2-years.gmi',
-null,
-'2023-02-26'
-);
-

code/common/db/src/main/resources/sql/migrations/01-domain.sql

@@ -1 +0,0 @@
-ALTER TABLE EC_DOMAIN MODIFY COLUMN IP VARCHAR(48);

code/common/db/src/test/java/nu/marginalia/db/DomainTypesTest.java

@@ -24,7 +24,7 @@ public class DomainTypesTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("sql/current/10-domain-type.sql")
+            .withInitScript("db/migration/V23_07_0_001__domain_type.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;

code/common/db/src/test/java/nu/marginalia/db/storage/FileStorageServiceTest.java

@@ -0,0 +1,154 @@
+package nu.marginalia.db.storage;
+
+import com.google.common.collect.Lists;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.db.storage.model.FileStorageBaseType;
+import nu.marginalia.db.storage.model.FileStorageType;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.parallel.Execution;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import static org.junit.Assert.*;
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+@Testcontainers
+@Execution(SAME_THREAD)
+@Tag("slow")
+public class FileStorageServiceTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_07_0_004__file_storage.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    static FileStorageService fileStorageService;
+
+    static List<Path> tempDirs = new ArrayList<>();
+
+    @BeforeAll
+    public static void setup() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+    }
+
+
+    @BeforeEach
+    public void setupEach() {
+        fileStorageService = new FileStorageService(dataSource);
+    }
+
+    @AfterEach
+    public void tearDownEach() {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.createStatement()) {
+            stmt.execute("DELETE FROM FILE_STORAGE");
+            stmt.execute("DELETE FROM FILE_STORAGE_BASE");
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @AfterAll
+    public static void teardown() {
+        dataSource.close();
+
+        Lists.reverse(tempDirs).forEach(path -> {
+            try {
+                System.out.println("Deleting " + path);
+                Files.delete(path);
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        });
+    }
+
+    private Path createTempDir() {
+        try {
+            Path dir = Files.createTempDirectory("file-storage-test");
+            tempDirs.add(dir);
+            return dir;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+    @Test
+    public void testCreateBase() throws SQLException, FileNotFoundException {
+        String name = "test-" + UUID.randomUUID();
+
+        var storage = new FileStorageService(dataSource);
+        var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false);
+
+        Assertions.assertEquals(name, base.name());
+        Assertions.assertEquals(FileStorageBaseType.SLOW, base.type());
+        Assertions.assertFalse(base.permitTemp());
+    }
+    @Test
+    public void testAllocateTempInNonPermitted() throws SQLException, FileNotFoundException {
+        String name = "test-" + UUID.randomUUID();
+
+        var storage = new FileStorageService(dataSource);
+
+        var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false);
+
+        try {
+            storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldFail");
+            fail();
+        }
+        catch (IllegalArgumentException ex) {} // ok
+        catch (Exception ex) {
+            ex.printStackTrace();
+            fail();
+        }
+    }
+
+    @Test
+    public void testAllocatePermanentInNonPermitted() throws SQLException, IOException {
+        String name = "test-" + UUID.randomUUID();
+
+        var storage = new FileStorageService(dataSource);
+
+        var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, false);
+
+        var created = storage.allocatePermanentStorage(base, "xyz", FileStorageType.CRAWL_DATA, "thisShouldSucceed");
+        tempDirs.add(created.asPath());
+
+        var actual = storage.getStorage(created.id());
+        Assertions.assertEquals(created, actual);
+    }
+
+    @Test
+    public void testAllocateTempInPermitted() throws IOException, SQLException {
+        String name = "test-" + UUID.randomUUID();
+
+        var storage = new FileStorageService(dataSource);
+
+        var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.SLOW, true);
+        var fileStorage = storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldSucceed");
+
+        Assertions.assertTrue(Files.exists(fileStorage.asPath()));
+        tempDirs.add(fileStorage.asPath());
+    }
+
+
+}

code/common/message-queue/build.gradle

@@ -0,0 +1,49 @@
+plugins {
+    id 'java'
+}
+
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(17))
+    }
+}
+
+dependencies {
+    implementation project(':code:common:service-client')
+    implementation project(':code:common:service-discovery')
+    implementation project(':code:common:db')
+
+    implementation libs.lombok
+    annotationProcessor libs.lombok
+
+    implementation libs.spark
+    implementation libs.guice
+    implementation libs.gson
+    implementation libs.rxjava
+
+    implementation libs.bundles.prometheus
+    implementation libs.bundles.slf4j
+    implementation libs.bucket4j
+
+    testImplementation libs.bundles.slf4j.test
+    implementation libs.bundles.mariadb
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+    testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
+    testImplementation 'org.testcontainers:mariadb:1.17.4'
+    testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+}
+
+test {
+    useJUnitPlatform()
+}
+
+task fastTests(type: Test) {
+    useJUnitPlatform {
+        excludeTags "slow"
+    }
+}

code/common/message-queue/readme.md

@@ -0,0 +1,100 @@
+# Message Queue
+
+Implements resilient message queueing for the application,
+as well as a finite state machine library backed by the
+message queue that enables long-running tasks that outlive
+the execution lifespan of the involved processes. 
+
+![Message States](msgstate.svg)
+
+The message queue is interacted with via the Inbox and Outbox classes.
+
+There are three types of inboxes;
+
+Name|Description
+---|---
+MqSingleShotInbox|A single message is received and then the inbox is closed.
+MqAsynchronousInbox|Messages are received asynchronously and can be processed in parallel.
+MqSynchronousInbox|Messages are received synchronously and will be processed in order; message processing can be aborted.
+
+A single outbox implementation exists, the `MqOutbox`, which implements multiple message sending strategies,
+including blocking and asynchronous paradigms.  Lower level access to the message queue itself is provided by the `MqPersistence` class.
+
+The inbox implementations as well as the outbox can be constructed via the `MessageQueueFactory` class. 
+
+## Message Queue State Machine (MQSM)
+
+The MQSM is a finite state machine that is backed by the message queue used to implement an Actor style paradigm. 
+
+The machine itself is defined through a class that extends the 'AbstractStateGraph'; with state transitions and
+names defined as implementations.
+
+Example:
+
+```java
+class ExampleStateMachine extends AbstractStateGraph {
+    
+    @GraphState(name = "INITIAL", next="GREET")
+    public void initial() {
+        return "World"; // passed to the next state
+    }
+
+    @GraphState(name = "GREET", next="COUNT-TO-FIVE")
+    public void greet(String name) {
+        System.out.println("Hello " + name);
+    }
+
+    @GraphState(name = "COUNT-TO-FIVE", next="END")
+    public void countToFive(Integer value) {
+        // value is passed from the previous state, since greet didn't pass a value,
+        // null will be the default.
+        
+        if (null == value) {
+            // jumps to the current state with a value of 0
+            transition("COUNT-TO-FIVE", 0);
+        }
+
+
+        System.out.println(++value);
+        if (value < 5) {
+            // Loops the current state until value = 5
+            transition("COUNT-TO-FIVE", value);
+        }
+        
+        if (value > 5) {
+            // demonstrates an error condition
+            error("Illegal value");
+        }
+        
+        // Default transition is to END
+    }
+    
+    @GraphState(name="END")
+    public void end() {
+        System.out.println("Done");
+    }
+}
+```
+
+Each method should ideally be idempotent, or at least be able to handle being called multiple times.
+It can not be assumed that the states are invoked within the same process, or even on the same machine,
+on the same day, etc.
+
+The usual considerations for writing deterministic Java code are advisable unless unavoidable; 
+all state must be local, don't iterate over hash maps, etc.
+
+### Create a state machine
+To create an ActorStateMachine from the above class, the following code can be used:
+
+```java
+ActorStateMachine actorStateMachine = new ActorStateMachine(
+        messageQueueFactory, 
+        actorInboxName, 
+        actorInstanceUUID,
+        new ExampleStateMachine());
+
+actorStateMachine.start();
+```
+
+The state machine will now run until it reaches the end state 
+and listen to messages on the inbox for state transitions.

code/common/message-queue/src/main/java/nu/marginalia/mq/MessageQueueFactory.java

@@ -0,0 +1,44 @@
+package nu.marginalia.mq;
+
+import nu.marginalia.mq.inbox.MqAsynchronousInbox;
+import nu.marginalia.mq.inbox.MqInboxIf;
+import nu.marginalia.mq.inbox.MqSingleShotInbox;
+import nu.marginalia.mq.inbox.MqSynchronousInbox;
+import nu.marginalia.mq.outbox.MqOutbox;
+import nu.marginalia.mq.persistence.MqPersistence;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.util.UUID;
+
+@Singleton
+public class MessageQueueFactory {
+    private final MqPersistence persistence;
+
+    @Inject
+    public MessageQueueFactory(MqPersistence persistence) {
+        this.persistence = persistence;
+    }
+
+    public MqSingleShotInbox createSingleShotInbox(String inboxName, UUID instanceUUID)
+    {
+        return new MqSingleShotInbox(persistence, inboxName, instanceUUID);
+    }
+
+
+    public MqAsynchronousInbox createAsynchronousInbox(String inboxName, UUID instanceUUID)
+    {
+        return new MqAsynchronousInbox(persistence, inboxName, instanceUUID);
+    }
+
+    public MqSynchronousInbox createSynchronousInbox(String inboxName, UUID instanceUUID)
+    {
+        return new MqSynchronousInbox(persistence, inboxName, instanceUUID);
+    }
+
+
+    public MqOutbox createOutbox(String inboxName, String outboxName, UUID instanceUUID)
+    {
+        return new MqOutbox(persistence, inboxName, outboxName, instanceUUID);
+    }
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/MqException.java

@@ -0,0 +1,11 @@
+package nu.marginalia.mq;
+
+public class MqException extends Exception {
+    public MqException(String message) {
+        super(message);
+    }
+
+    public MqException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/MqMessage.java

@@ -0,0 +1,11 @@
+package nu.marginalia.mq;
+
+public record MqMessage(
+        long msgId,
+        long relatedId,
+        String function,
+        String payload,
+        MqMessageState state,
+        boolean expectsResponse
+) {
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/MqMessageState.java

@@ -0,0 +1,14 @@
+package nu.marginalia.mq;
+
+public enum MqMessageState {
+    /** The message is new and has not yet been acknowledged by the recipient */
+    NEW,
+    /** The message has been acknowledged by the recipient */
+    ACK,
+    /** The message has been processed successfully by the recipient */
+    OK,
+    /** The message processing has failed */
+    ERR,
+    /** The message did not reach a terminal state within the TTL */
+    DEAD
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/inbox/MqAsynchronousInbox.java

@@ -0,0 +1,226 @@
+package nu.marginalia.mq.inbox;
+
+import nu.marginalia.mq.MqMessage;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.persistence.MqPersistence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+/** Message queue inbox that spawns news threads for each message */
+public class MqAsynchronousInbox implements MqInboxIf {
+    private final Logger logger = LoggerFactory.getLogger(MqAsynchronousInbox.class);
+
+    private final String inboxName;
+    private final String instanceUUID;
+    private final ExecutorService threadPool;
+    private final MqPersistence persistence;
+
+    private volatile boolean run = true;
+
+    private final int pollIntervalMs = Integer.getInteger("mq.inbox.poll-interval-ms", 100);
+    private final int maxPollCount = Integer.getInteger("mq.inbox.max-poll-count", 10);
+    private final List<MqSubscription> eventSubscribers = new ArrayList<>();
+    private final LinkedBlockingQueue<MqMessage> queue = new LinkedBlockingQueue<>(32);
+
+    private Thread pollDbThread;
+    private Thread notifyThread;
+
+    public MqAsynchronousInbox(MqPersistence persistence,
+                               String inboxName,
+                               UUID instanceUUID)
+    {
+        this(persistence, inboxName, instanceUUID, Executors.newCachedThreadPool());
+    }
+
+    public MqAsynchronousInbox(MqPersistence persistence,
+                               String inboxName,
+                               UUID instanceUUID,
+                               ExecutorService executorService)
+    {
+        this.threadPool = executorService;
+        this.persistence = persistence;
+        this.inboxName = inboxName;
+        this.instanceUUID = instanceUUID.toString();
+    }
+
+    /** Subscribe to messages on this inbox. Must be run before start()! */
+    @Override
+    public void subscribe(MqSubscription subscription) {
+        eventSubscribers.add(subscription);
+    }
+
+    /** Start receiving messages. <p>
+     * <b>Note:</b> Subscribe to messages before calling this method.
+     * </p> */
+    @Override
+    public void start() {
+        run = true;
+
+        if (eventSubscribers.isEmpty()) {
+            logger.error("No subscribers for inbox {}, registering shredder", inboxName);
+        }
+
+        // Add a final handler that fails any message that is not handled
+        eventSubscribers.add(new MqInboxShredder());
+
+        pollDbThread = new Thread(this::pollDb, "mq-inbox-update-thread:"+inboxName);
+        pollDbThread.setDaemon(true);
+        pollDbThread.start();
+
+        notifyThread = new Thread(this::notifySubscribers, "mq-inbox-notify-thread:"+inboxName);
+        notifyThread.setDaemon(true);
+        notifyThread.start();
+    }
+
+    /** Stop receiving messages and shut down all threads */
+    @Override
+    public void stop() throws InterruptedException {
+        if (!run)
+            return;
+
+        logger.info("Shutting down inbox {}", inboxName);
+
+        run = false;
+        pollDbThread.join();
+        notifyThread.join();
+
+        threadPool.shutdownNow();
+
+        while (!threadPool.awaitTermination(5, TimeUnit.SECONDS));
+    }
+
+    private void notifySubscribers() {
+        try {
+            while (run) {
+
+                MqMessage msg = queue.poll(pollIntervalMs, TimeUnit.MILLISECONDS);
+
+                if (msg == null)
+                    continue;
+
+                logger.info("Notifying subscribers of message {}", msg.msgId());
+
+                boolean handled = false;
+
+                for (var eventSubscriber : eventSubscribers) {
+                    if (eventSubscriber.filter(msg)) {
+                        handleMessageWithSubscriber(eventSubscriber, msg);
+                        handled = true;
+                        break;
+                    }
+                }
+
+                if (!handled) {
+                    logger.error("No subscriber wanted to handle message {}", msg.msgId());
+                }
+            }
+        }
+        catch (InterruptedException ex) {
+            logger.error("MQ inbox notify thread interrupted", ex);
+        }
+    }
+
+    private void handleMessageWithSubscriber(MqSubscription subscriber, MqMessage msg) {
+
+        if (msg.expectsResponse()) {
+            threadPool.execute(() -> respondToMessage(subscriber, msg));
+        }
+        else {
+            threadPool.execute(() -> acknowledgeNotification(subscriber, msg));
+        }
+    }
+
+    private void respondToMessage(MqSubscription subscriber, MqMessage msg) {
+        try {
+            final var rsp = subscriber.onRequest(msg);
+            sendResponse(msg, rsp.state(), rsp.message());
+        } catch (Exception ex) {
+            logger.error("Message Queue subscriber threw exception", ex);
+            sendResponse(msg, MqMessageState.ERR);
+        }
+    }
+
+    private void acknowledgeNotification(MqSubscription subscriber, MqMessage msg) {
+        try {
+            subscriber.onNotification(msg);
+            updateMessageState(msg, MqMessageState.OK);
+        } catch (Exception ex) {
+            logger.error("Message Queue subscriber threw exception", ex);
+            updateMessageState(msg, MqMessageState.ERR);
+        }
+    }
+
+    private void sendResponse(MqMessage msg, MqMessageState state) {
+        try {
+            persistence.updateMessageState(msg.msgId(), state);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to update message state", ex);
+        }
+    }
+
+    private void updateMessageState(MqMessage msg, MqMessageState state) {
+        try {
+            persistence.updateMessageState(msg.msgId(), state);
+        }
+        catch (SQLException ex2) {
+            logger.error("Failed to update message state", ex2);
+        }
+    }
+
+    private void sendResponse(MqMessage msg, MqMessageState mqMessageState, String response) {
+        try {
+            persistence.sendResponse(msg.msgId(), mqMessageState, response);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to update message state", ex);
+        }
+    }
+
+    private void pollDb() {
+        try {
+            for (long tick = 1; run; tick++) {
+
+                queue.addAll(pollInbox(tick));
+
+                TimeUnit.MILLISECONDS.sleep(pollIntervalMs);
+            }
+        }
+        catch (InterruptedException ex) {
+            logger.error("MQ inbox update thread interrupted", ex);
+        }
+    }
+
+     private Collection<MqMessage> pollInbox(long tick) {
+        try {
+            return persistence.pollInbox(inboxName, instanceUUID, tick, maxPollCount);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to poll inbox", ex);
+            return List.of();
+        }
+    }
+
+     /** Retrieve the last N messages from the inbox. */
+    @Override
+    public List<MqMessage> replay(int lastN) {
+        try {
+            return persistence.lastNMessages(inboxName, lastN);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to replay inbox", ex);
+            return List.of();
+        }
+    }
+
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxIf.java

@@ -0,0 +1,15 @@
+package nu.marginalia.mq.inbox;
+
+import nu.marginalia.mq.MqMessage;
+
+import java.util.List;
+
+public interface MqInboxIf {
+    void subscribe(MqSubscription subscription);
+
+    void start();
+
+    void stop() throws InterruptedException;
+
+    List<MqMessage> replay(int lastN);
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxResponse.java

@@ -0,0 +1,22 @@
+package nu.marginalia.mq.inbox;
+
+import nu.marginalia.mq.MqMessageState;
+
+public record MqInboxResponse(String message, MqMessageState state) {
+
+    public static MqInboxResponse ok(String message) {
+        return new MqInboxResponse(message, MqMessageState.OK);
+    }
+
+    public static MqInboxResponse ok() {
+        return new MqInboxResponse("", MqMessageState.OK);
+    }
+
+    public static MqInboxResponse err(String message) {
+        return new MqInboxResponse(message, MqMessageState.ERR);
+    }
+
+    public static MqInboxResponse err() {
+        return new MqInboxResponse("", MqMessageState.ERR);
+    }
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/inbox/MqInboxShredder.java

@@ -0,0 +1,29 @@
+package nu.marginalia.mq.inbox;
+
+import nu.marginalia.mq.MqMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class MqInboxShredder implements MqSubscription {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    public MqInboxShredder() {
+    }
+
+    @Override
+    public boolean filter(MqMessage rawMessage) {
+        return true;
+    }
+
+    @Override
+    public MqInboxResponse onRequest(MqMessage msg) {
+        logger.warn("Unhandled message {}", msg.msgId());
+        return MqInboxResponse.err();
+    }
+
+    @Override
+    public void onNotification(MqMessage msg) {
+        logger.warn("Unhandled message {}", msg.msgId());
+    }
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSingleShotInbox.java

@@ -0,0 +1,100 @@
+package nu.marginalia.mq.inbox;
+
+import lombok.SneakyThrows;
+import nu.marginalia.mq.MqMessage;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.persistence.MqPersistence;
+
+import java.sql.SQLException;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+/** A single-shot inbox that can be used to wait for a single message
+ *  to arrive in an inbox, and then reply to that message
+ */
+public class MqSingleShotInbox {
+
+    private final String inboxName;
+    private final String instanceUUID;
+    private final MqPersistence persistence;
+
+    public MqSingleShotInbox(MqPersistence persistence,
+                             String inboxName,
+                             UUID instanceUUID
+                             ) {
+        this.inboxName = inboxName;
+        this.instanceUUID = instanceUUID.toString();
+        this.persistence = persistence;
+    }
+
+    /** Wait for a message to arrive in the specified inbox, up to the specified timeout.
+     *
+     *  @param timeout The timeout
+     *  @param unit The time unit
+     *  @return The message, or empty if no message arrived before the timeout
+     */
+    public Optional<MqMessage> waitForMessage(long timeout, TimeUnit unit) throws InterruptedException, SQLException {
+        final long deadline = System.currentTimeMillis() + unit.toMillis(timeout);
+
+        for (int i = 0;; i++) {
+            if (System.currentTimeMillis() >= deadline) {
+                return Optional.empty();
+            }
+
+            var messages = persistence.pollInbox(inboxName, instanceUUID, i, 1);
+
+            if (messages.size() > 0) {
+                return Optional.of(messages.iterator().next());
+            }
+
+            TimeUnit.SECONDS.sleep(1);
+        }
+    }
+
+
+    /** Steal a message from the inbox, and change the owner to this instance.  This is useful
+     * for resuming an aborted process.  This should be done judiciously, only in cases we're certain
+     * that the original owner is no longer running as it may cause duplicate processing, race
+     * conditions, etc.
+     * <p>
+     *  @param predicate A predicate that must be true for the message to be stolen
+     *  @return The stolen message, or empty if no message was stolen
+     */
+    @SneakyThrows
+    public Optional<MqMessage> stealMessage(Predicate<MqMessage> predicate) {
+        for (var message : persistence.eavesdrop(inboxName, 5)) {
+            if (predicate.test(message)) {
+                persistence.changeOwner(message.msgId(), instanceUUID, -1);
+                return Optional.of(message);
+            }
+        }
+
+        return Optional.empty();
+    }
+
+    /** Send a response to the specified message. If the original message has no response inbox,
+     * the original message will be marked as OK instead.
+     *
+     *  @param originalMessage The original message
+     *  @param response The response
+     */
+    public void sendResponse(MqMessage originalMessage, MqInboxResponse response) {
+        try {
+            if (!originalMessage.expectsResponse()) {
+                // If the original message doesn't expect a response, we can just mark it as OK,
+                // since the sendResponse method will fail explosively since it can't insert a response
+                // to a non-existent inbox.
+
+                persistence.updateMessageState(originalMessage.msgId(), MqMessageState.OK);
+            }
+            else {
+                persistence.sendResponse(originalMessage.msgId(), response.state(), response.message());
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSubscription.java

@@ -0,0 +1,14 @@
+package nu.marginalia.mq.inbox;
+
+import nu.marginalia.mq.MqMessage;
+
+public interface MqSubscription {
+    /** Return true if this subscription should handle the message. */
+    boolean filter(MqMessage rawMessage);
+
+    /** Handle the message and return a response. */
+    MqInboxResponse onRequest(MqMessage msg);
+
+    /** Handle a message with no reply address */
+    void onNotification(MqMessage msg);
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/inbox/MqSynchronousInbox.java

@@ -0,0 +1,222 @@
+package nu.marginalia.mq.inbox;
+
+import nu.marginalia.mq.MqMessage;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.persistence.MqPersistence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/** Message queue inbox that responds to a single message at a time
+ * within the polling thread
+ */
+public class MqSynchronousInbox implements MqInboxIf {
+    private final Logger logger = LoggerFactory.getLogger(MqSynchronousInbox.class);
+
+    private final String inboxName;
+    private final String instanceUUID;
+    private final MqPersistence persistence;
+
+    private volatile boolean run = true;
+
+    private final int pollIntervalMs = Integer.getInteger("mq.inbox.poll-interval-ms", 100);
+    private final List<MqSubscription> eventSubscribers = new ArrayList<>();
+
+    private Thread pollDbThread;
+    private ExecutorService executorService = Executors.newSingleThreadExecutor();
+
+    public MqSynchronousInbox(MqPersistence persistence,
+                              String inboxName,
+                              UUID instanceUUID)
+    {
+        this.persistence = persistence;
+        this.inboxName = inboxName;
+        this.instanceUUID = instanceUUID.toString();
+    }
+
+    /** Subscribe to messages on this inbox. Must be run before start()! */
+    @Override
+    public void subscribe(MqSubscription subscription) {
+        eventSubscribers.add(subscription);
+    }
+
+    /** Start receiving messages. <p>
+     * <b>Note:</b> Subscribe to messages before calling this method.
+     * </p> */
+    @Override
+    public void start() {
+        run = true;
+
+        if (eventSubscribers.isEmpty()) {
+            logger.error("No subscribers for inbox {}, registering shredder", inboxName);
+        }
+
+        // Add a final handler that fails any message that is not handled
+        eventSubscribers.add(new MqInboxShredder());
+
+        pollDbThread = new Thread(this::pollDb, "mq-inbox-update-thread:"+inboxName);
+        pollDbThread.setDaemon(true);
+        pollDbThread.start();
+    }
+
+    /** Stop receiving messages and shut down all threads */
+    @Override
+    public void stop() throws InterruptedException {
+        if (!run)
+            return;
+
+        logger.info("Shutting down inbox {}", inboxName);
+
+        run = false;
+        pollDbThread.join();
+        executorService.shutdown();
+        executorService.awaitTermination(10, TimeUnit.SECONDS);
+
+    }
+
+    private void handleMessageWithSubscriber(MqSubscription subscriber, MqMessage msg) {
+
+        if (msg.expectsResponse()) {
+            respondToMessage(subscriber, msg);
+        }
+        else {
+            acknowledgeNotification(subscriber, msg);
+        }
+    }
+
+    private void respondToMessage(MqSubscription subscriber, MqMessage msg) {
+        try {
+            final var rsp = subscriber.onRequest(msg);
+            sendResponse(msg, rsp.state(), rsp.message());
+        } catch (Exception ex) {
+            logger.error("Message Queue subscriber threw exception", ex);
+            sendResponse(msg, MqMessageState.ERR);
+        }
+    }
+
+    private void acknowledgeNotification(MqSubscription subscriber, MqMessage msg) {
+        try {
+            subscriber.onNotification(msg);
+            updateMessageState(msg, MqMessageState.OK);
+        }
+        catch (Exception ex) {
+            logger.error("Message Queue subscriber threw exception", ex);
+            updateMessageState(msg, MqMessageState.ERR);
+        }
+    }
+
+    private void sendResponse(MqMessage msg, MqMessageState state) {
+        try {
+            persistence.updateMessageState(msg.msgId(), state);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to update message state", ex);
+        }
+    }
+
+    private void updateMessageState(MqMessage msg, MqMessageState state) {
+        try {
+            persistence.updateMessageState(msg.msgId(), state);
+        }
+        catch (SQLException ex2) {
+            logger.error("Failed to update message state", ex2);
+        }
+    }
+
+    private void sendResponse(MqMessage msg, MqMessageState mqMessageState, String response) {
+        try {
+            persistence.sendResponse(msg.msgId(), mqMessageState, response);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to update message state", ex);
+        }
+    }
+
+    private volatile java.util.concurrent.Future<?> currentTask = null;
+    public void pollDb() {
+        try {
+            for (long tick = 1; run; tick++) {
+
+                var messages = pollInbox(tick);
+
+                for (var msg : messages) {
+                    // Handle message in a separate thread but wait for that thread, so we can interrupt that thread
+                    // without interrupting the polling thread and shutting down the inbox completely
+                    try {
+                        currentTask = executorService.submit(() -> handleMessage(msg));
+                        currentTask.get();
+                    }
+                    catch (Exception ex) {
+                        logger.error("Inbox task was aborted");
+                    }
+                    finally {
+                        currentTask = null;
+                    }
+                }
+
+                if (messages.isEmpty()) {
+                    TimeUnit.MILLISECONDS.sleep(pollIntervalMs);
+                }
+            }
+        }
+        catch (InterruptedException ex) {
+            logger.error("MQ inbox update thread interrupted", ex);
+        }
+    }
+
+    /** Attempt to abort the current task using an interrupt */
+    public void abortCurrentTask() {
+        var task = currentTask; // capture the value to avoid race conditions with the
+                                // polling thread between the check and the interrupt
+        if (task != null) {
+            task.cancel(true);
+        }
+    }
+
+
+    private void handleMessage(MqMessage msg) {
+        logger.info("Notifying subscribers of msg {}", msg.msgId());
+
+        boolean handled = false;
+
+        for (var eventSubscriber : eventSubscribers) {
+            if (eventSubscriber.filter(msg)) {
+                handleMessageWithSubscriber(eventSubscriber, msg);
+                handled = true;
+                break;
+            }
+        }
+
+        if (!handled) {
+            logger.error("No subscriber wanted to handle msg {}", msg.msgId());
+        }
+    }
+
+    private Collection<MqMessage> pollInbox(long tick) {
+        try {
+            return persistence.pollInbox(inboxName, instanceUUID, tick, 1);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to poll inbox", ex);
+            return List.of();
+        }
+    }
+
+    /** Retrieve the last N messages from the inbox. */
+    @Override
+    public List<MqMessage> replay(int lastN) {
+        try {
+            return persistence.lastNMessages(inboxName, lastN);
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to replay inbox", ex);
+            return List.of();
+        }
+    }
+
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java

@@ -0,0 +1,172 @@
+package nu.marginalia.mq.outbox;
+
+import nu.marginalia.mq.MqMessage;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.persistence.MqPersistence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class MqOutbox {
+    private final Logger logger = LoggerFactory.getLogger(MqOutbox.class);
+    private final MqPersistence persistence;
+    private final String inboxName;
+    private final String replyInboxName;
+    private final String instanceUUID;
+
+    private final ConcurrentHashMap<Long, MqMessage> pendingResponses = new ConcurrentHashMap<>();
+
+    private final int pollIntervalMs = Integer.getInteger("mq.outbox.poll-interval-ms", 100);
+    private final int maxPollCount = Integer.getInteger("mq.outbox.max-poll-count", 10);
+    private final Thread pollThread;
+
+    private volatile boolean run = true;
+
+    public MqOutbox(MqPersistence persistence,
+                    String inboxName,
+                    String outboxName,
+                    UUID instanceUUID) {
+        this.persistence = persistence;
+
+        this.inboxName = inboxName;
+        this.replyInboxName = outboxName + "//" + inboxName;
+        this.instanceUUID = instanceUUID.toString();
+
+        pollThread = new Thread(this::poll, "mq-outbox-poll-thread:" + inboxName);
+        pollThread.setDaemon(true);
+        pollThread.start();
+    }
+
+    public void stop() throws InterruptedException {
+        if (!run)
+            return;
+
+        logger.info("Shutting down outbox {}", inboxName);
+
+        run = false;
+        pollThread.join();
+    }
+
+    private void poll() {
+        try {
+            for (long id = 1; run; id++) {
+                pollDb(id);
+
+                TimeUnit.MILLISECONDS.sleep(pollIntervalMs);
+            }
+        } catch (InterruptedException ex) {
+            logger.error("Outbox poll thread interrupted", ex);
+        }
+    }
+
+    private void pollDb(long tick) {
+        try {
+            var updates = persistence.pollReplyInbox(replyInboxName, instanceUUID, tick, maxPollCount);
+
+            for (var message : updates) {
+                pendingResponses.put(message.relatedId(), message);
+            }
+
+            if (updates.isEmpty())
+                return;
+
+            logger.info("Notifying {} pending responses", pendingResponses.size());
+
+            synchronized (pendingResponses) {
+                pendingResponses.notifyAll();
+            }
+        }
+        catch (SQLException ex) {
+            logger.error("Failed to poll inbox", ex);
+        }
+
+    }
+
+    /** Send a message and wait for a response. */
+    public MqMessage send(String function, String payload) throws Exception {
+        final long id = sendAsync(function, payload);
+
+        return waitResponse(id);
+    }
+
+    /** Send a message asynchronously, without waiting for a response.
+     * <br>
+     * Use waitResponse(id) or pollResponse(id) to fetch the response.  */
+    public long sendAsync(String function, String payload) throws Exception {
+        return persistence.sendNewMessage(inboxName, replyInboxName, null, function, payload, null);
+    }
+
+    /** Blocks until a response arrives for the given message id (possibly forever) */
+    public MqMessage waitResponse(long id) throws Exception {
+        synchronized (pendingResponses) {
+            while (!pendingResponses.containsKey(id)) {
+                pendingResponses.wait(100);
+            }
+
+            var msg = pendingResponses.remove(id);
+            // Mark the response as OK so it can be cleaned up
+            persistence.updateMessageState(msg.msgId(), MqMessageState.OK);
+
+            return msg;
+        }
+    }
+
+
+    /** Blocks until a response arrives for the given message id or the timeout passes.
+     * <p>
+     * @throws TimeoutException if the timeout passes before a response arrives.
+     * @throws InterruptedException if the thread is interrupted while waiting.
+     */
+    public MqMessage waitResponse(long id, int timeout, TimeUnit unit) throws TimeoutException, SQLException, InterruptedException {
+        long deadline = System.currentTimeMillis() + unit.toMillis(timeout);
+
+        synchronized (pendingResponses) {
+            while (!pendingResponses.containsKey(id)) {
+                if (System.currentTimeMillis() > deadline)
+                    throw new TimeoutException("Timeout waiting for response");
+
+                pendingResponses.wait(100);
+            }
+
+            var msg = pendingResponses.remove(id);
+            // Mark the response as OK so it can be cleaned up
+            persistence.updateMessageState(msg.msgId(), MqMessageState.OK);
+
+            return msg;
+        }
+    }
+
+    /** Polls for a response for the given message id. */
+    public Optional<MqMessage> pollResponse(long id) throws SQLException {
+        // no need to sync here if we aren't going to wait()
+        var response = pendingResponses.get(id);
+
+        if (response != null) {
+            // Mark the response as OK so it can be cleaned up
+            persistence.updateMessageState(response.msgId(), MqMessageState.OK);
+        }
+        return Optional.ofNullable(response);
+    }
+
+    public long sendNotice(String function, String payload) throws Exception {
+        return persistence.sendNewMessage(inboxName, null, null, function, payload, null);
+    }
+    public long sendNotice(long relatedId, String function, String payload) throws Exception {
+        return persistence.sendNewMessage(inboxName, null, relatedId, function, payload, null);
+    }
+
+    public void flagAsBad(long id) throws SQLException {
+        persistence.updateMessageState(id, MqMessageState.ERR);
+    }
+
+    public void flagAsDead(long id) throws SQLException {
+        persistence.updateMessageState(id, MqMessageState.DEAD);
+    }
+
+}

code/common/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java

@@ -0,0 +1,487 @@
+package nu.marginalia.mq.persistence;
+
+import com.google.common.collect.Lists;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.MqMessage;
+
+import javax.annotation.Nullable;
+import java.sql.SQLException;
+import java.time.Duration;
+import java.util.*;
+
+import static nu.marginalia.mq.MqMessageState.NEW;
+
+/** A persistence layer for the message queue.
+ *  <p>
+ *  All storage operations must be done through this class.
+ */
+@Singleton
+public class MqPersistence {
+    private final HikariDataSource dataSource;
+
+    @Inject
+    public MqPersistence(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    /**
+     * Adds a new message to the message queue.
+     *
+     * @param recipientInboxName The recipient's inbox name
+     * @param senderInboxName    (nullable) The sender's inbox name. Only needed if a reply is expected. If null, the message is not expected to be replied to.
+     * @param relatedMessageId   (nullable) The id of the message this message is related to. If null, the message is not related to any other message.
+     * @param function           The function to call
+     * @param payload            The payload to send, typically JSON.
+     * @param ttl                (nullable) The time to live of the message, in seconds. If null, the message will never set to DEAD.
+     * @return The id of the message
+     */
+    public long sendNewMessage(String recipientInboxName,
+                               @Nullable
+                               String senderInboxName,
+                               Long relatedMessageId,
+                               String function,
+                               String payload,
+                               @Nullable Duration ttl
+                               ) throws Exception {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     INSERT INTO MESSAGE_QUEUE(RECIPIENT_INBOX, SENDER_INBOX, RELATED_ID, FUNCTION, PAYLOAD, TTL)
+                     VALUES(?, ?, ?, ?, ?, ?)
+                     """);
+             var lastIdQuery = conn.prepareStatement("SELECT LAST_INSERT_ID()")) {
+
+            stmt.setString(1, recipientInboxName);
+
+            if (senderInboxName == null) stmt.setNull(2, java.sql.Types.VARCHAR);
+            else stmt.setString(2, senderInboxName);
+
+            // Translate null to -1, as 0 is a valid id
+            stmt.setLong(3, Objects.requireNonNullElse(relatedMessageId, -1L));
+
+            stmt.setString(4, function);
+            stmt.setString(5, payload);
+            if (ttl == null) stmt.setNull(6, java.sql.Types.BIGINT);
+            else stmt.setLong(6, ttl.toSeconds());
+
+            stmt.executeUpdate();
+
+            if (!conn.getAutoCommit())
+                conn.commit();
+
+            var rsp = lastIdQuery.executeQuery();
+
+            if (!rsp.next()) {
+                throw new IllegalStateException("No last insert id");
+            }
+
+            return rsp.getLong(1);
+        }
+    }
+
+    /** Modifies the state of a message by id.
+     * <p>
+     * If the state is 'NEW', ownership information will be stripped to avoid creating
+     * a broken message that can't be dequeued because it has an owner.
+     *
+     * @param id The id of the message
+     * @param mqMessageState The new state
+     * */
+    public void updateMessageState(long id, MqMessageState mqMessageState) throws SQLException {
+        if (NEW == mqMessageState) {
+            reinitializeMessage(id);
+            return;
+        }
+
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     UPDATE MESSAGE_QUEUE
+                     SET STATE=?, UPDATED_TIME=CURRENT_TIMESTAMP(6)
+                     WHERE ID=?
+                     """)) {
+
+            stmt.setString(1, mqMessageState.name());
+            stmt.setLong(2, id);
+
+            if (stmt.executeUpdate() != 1) {
+                throw new IllegalArgumentException("No rows updated");
+            }
+
+            if (!conn.getAutoCommit())
+                conn.commit();
+        }
+    }
+
+    /** Sets the message to 'NEW' state and removes any owner */
+    private void reinitializeMessage(long id) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     UPDATE MESSAGE_QUEUE
+                     SET STATE='NEW',
+                         OWNER_INSTANCE=NULL,
+                         OWNER_TICK=NULL,
+                         UPDATED_TIME=CURRENT_TIMESTAMP(6)
+                     WHERE ID=?
+                     """)) {
+
+            stmt.setLong(1, id);
+
+            if (stmt.executeUpdate() != 1) {
+                throw new IllegalArgumentException("No rows updated");
+            }
+
+            if (!conn.getAutoCommit())
+                conn.commit();
+        }
+    }
+
+    /** Creates a new message in the queue referencing as a reply to an existing message
+     *  This message will have it's RELATED_ID set to the original message's ID.
+     */
+    public long sendResponse(long id, MqMessageState mqMessageState, String message) throws SQLException {
+        try (var conn = dataSource.getConnection()) {
+            conn.setAutoCommit(false);
+
+            try (var updateState = conn.prepareStatement("""
+                    UPDATE MESSAGE_QUEUE
+                    SET STATE=?, UPDATED_TIME=CURRENT_TIMESTAMP(6)
+                    WHERE ID=?
+                    """);
+                 var addResponse = conn.prepareStatement("""
+                         INSERT INTO MESSAGE_QUEUE(RECIPIENT_INBOX, RELATED_ID, FUNCTION, PAYLOAD)
+                         SELECT SENDER_INBOX, ID, ?, ?
+                         FROM MESSAGE_QUEUE
+                         WHERE ID=? AND SENDER_INBOX IS NOT NULL
+                         """);
+                 var lastIdQuery = conn.prepareStatement("SELECT LAST_INSERT_ID()")
+            ) {
+
+                updateState.setString(1, mqMessageState.name());
+                updateState.setLong(2, id);
+                if (updateState.executeUpdate() != 1) {
+                    throw new IllegalArgumentException("No rows updated");
+                }
+
+                addResponse.setString(1, "REPLY");
+                addResponse.setString(2, message);
+                addResponse.setLong(3, id);
+                if (addResponse.executeUpdate() != 1) {
+                    throw new IllegalArgumentException("No rows updated");
+                }
+
+                var rsp = lastIdQuery.executeQuery();
+                if (!rsp.next()) {
+                    throw new IllegalStateException("No last insert id");
+                }
+                long newId = rsp.getLong(1);
+
+                conn.commit();
+
+                return newId;
+            } catch (SQLException|IllegalStateException|IllegalArgumentException ex) {
+                conn.rollback();
+                throw ex;
+            } finally {
+                conn.setAutoCommit(true);
+            }
+        }
+    }
+
+
+    /** Marks unclaimed messages addressed to this inbox with instanceUUID and tick,
+     * then returns the number of messages marked.  This is an atomic operation that
+     * ensures that messages aren't double processed.
+     */
+    private int markInboxMessages(String inboxName, String instanceUUID, long tick, int n) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var updateStmt = conn.prepareStatement("""
+                     UPDATE MESSAGE_QUEUE
+                     SET OWNER_INSTANCE=?, OWNER_TICK=?, UPDATED_TIME=CURRENT_TIMESTAMP(6), STATE='ACK'
+                     WHERE RECIPIENT_INBOX=?
+                     AND OWNER_INSTANCE IS NULL AND STATE='NEW'
+                     ORDER BY ID ASC
+                     LIMIT ?
+                     """);
+        ) {
+            updateStmt.setString(1, instanceUUID);
+            updateStmt.setLong(2, tick);
+            updateStmt.setString(3, inboxName);
+            updateStmt.setInt(4, n);
+            var ret = updateStmt.executeUpdate();
+            if (!conn.getAutoCommit())
+                conn.commit();
+            return ret;
+        }
+    }
+
+    /** Return up to n unprocessed messages from the specified inbox that are in states 'NEW' or 'ACK'
+     * without updating their ownership information
+     */
+    public Collection<MqMessage> eavesdrop(String inboxName, int n) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var queryStmt = conn.prepareStatement("""
+                     SELECT
+                        ID,
+                        RELATED_ID,
+                        FUNCTION,
+                        PAYLOAD,
+                        STATE,
+                        SENDER_INBOX IS NOT NULL AS EXPECTS_RESPONSE
+                     FROM MESSAGE_QUEUE
+                     WHERE STATE IN ('NEW', 'ACK')
+                     AND RECIPIENT_INBOX=?
+                     LIMIT ?
+                     """)
+        )
+        {
+        queryStmt.setString(1, inboxName);
+        queryStmt.setInt(2, n);
+        var rs = queryStmt.executeQuery();
+
+        List<MqMessage> messages = new ArrayList<>(n);
+
+        while (rs.next()) {
+            long msgId = rs.getLong("ID");
+            long relatedId = rs.getLong("RELATED_ID");
+
+            String function = rs.getString("FUNCTION");
+            String payload = rs.getString("PAYLOAD");
+
+            MqMessageState state = MqMessageState.valueOf(rs.getString("STATE"));
+            boolean expectsResponse = rs.getBoolean("EXPECTS_RESPONSE");
+
+            var msg = new MqMessage(msgId, relatedId, function, payload, state, expectsResponse);
+
+            messages.add(msg);
+        }
+
+        return messages;
+        }
+
+    }
+
+    /** Returns the message with the specified ID
+     *
+     * @throws SQLException if there is a problem with the database
+     * @throws IllegalArgumentException if the message doesn't exist
+     */
+    public MqMessage getMessage(long id) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var queryStmt = conn.prepareStatement("""
+                     SELECT
+                        ID,
+                        RELATED_ID,
+                        FUNCTION,
+                        PAYLOAD,
+                        STATE,
+                        SENDER_INBOX IS NOT NULL AS EXPECTS_RESPONSE
+                     FROM MESSAGE_QUEUE
+                     WHERE ID=?
+                     """)
+        )
+        {
+            queryStmt.setLong(1, id);
+            var rs = queryStmt.executeQuery();
+
+            if (rs.next()) {
+                long msgId = rs.getLong("ID");
+                long relatedId = rs.getLong("RELATED_ID");
+
+                String function = rs.getString("FUNCTION");
+                String payload = rs.getString("PAYLOAD");
+
+                MqMessageState state = MqMessageState.valueOf(rs.getString("STATE"));
+                boolean expectsResponse = rs.getBoolean("EXPECTS_RESPONSE");
+
+                return new MqMessage(msgId, relatedId, function, payload, state, expectsResponse);
+            }
+        }
+
+        throw new IllegalArgumentException("No message with id " + id);
+    }
+    /**  Marks unclaimed messages addressed to this inbox with instanceUUID and tick,
+     * then returns these messages.
+     */
+    public Collection<MqMessage> pollInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException {
+
+        // Mark new messages as claimed
+        int expected = markInboxMessages(inboxName, instanceUUID, tick, n);
+        if (expected == 0) {
+            return Collections.emptyList();
+        }
+
+        // Then fetch the messages that were marked
+        try (var conn = dataSource.getConnection();
+             var queryStmt = conn.prepareStatement("""
+                     SELECT
+                        ID,
+                        RELATED_ID,
+                        FUNCTION,
+                        PAYLOAD,
+                        STATE,
+                        SENDER_INBOX IS NOT NULL AS EXPECTS_RESPONSE
+                     FROM MESSAGE_QUEUE
+                     WHERE OWNER_INSTANCE=? AND OWNER_TICK=?
+                     """)
+        ) {
+            queryStmt.setString(1, instanceUUID);
+            queryStmt.setLong(2, tick);
+            var rs = queryStmt.executeQuery();
+
+            List<MqMessage> messages = new ArrayList<>(expected);
+
+            while (rs.next()) {
+                long msgId = rs.getLong("ID");
+                long relatedId = rs.getLong("RELATED_ID");
+
+                String function = rs.getString("FUNCTION");
+                String payload = rs.getString("PAYLOAD");
+
+                MqMessageState state = MqMessageState.valueOf(rs.getString("STATE"));
+                boolean expectsResponse = rs.getBoolean("EXPECTS_RESPONSE");
+
+                var msg = new MqMessage(msgId, relatedId, function, payload, state, expectsResponse);
+
+                messages.add(msg);
+            }
+
+            return messages;
+        }
+
+    }
+
+
+    /**  Marks unclaimed messages addressed to this inbox with instanceUUID and tick,
+     * then returns these messages.
+     */
+    public Collection<MqMessage> pollReplyInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException {
+
+        // Mark new messages as claimed
+        int expected = markInboxMessages(inboxName, instanceUUID, tick, n);
+        if (expected == 0) {
+            return Collections.emptyList();
+        }
+
+        // Then fetch the messages that were marked
+        try (var conn = dataSource.getConnection();
+             var queryStmt = conn.prepareStatement("""
+                     SELECT SELF.ID, SELF.RELATED_ID, SELF.FUNCTION, SELF.PAYLOAD, PARENT.STATE FROM MESSAGE_QUEUE SELF
+                     LEFT JOIN MESSAGE_QUEUE PARENT ON SELF.RELATED_ID=PARENT.ID
+                     WHERE SELF.OWNER_INSTANCE=? AND SELF.OWNER_TICK=?
+                     """)
+        ) {
+            queryStmt.setString(1, instanceUUID);
+            queryStmt.setLong(2, tick);
+            var rs = queryStmt.executeQuery();
+
+            List<MqMessage> messages = new ArrayList<>(expected);
+
+            while (rs.next()) {
+                long msgId = rs.getLong(1);
+                long relatedId = rs.getLong(2);
+
+                String function = rs.getString(3);
+                String payload = rs.getString(4);
+
+                MqMessageState state = MqMessageState.valueOf(rs.getString(5));
+
+                var msg = new MqMessage(msgId, relatedId, function, payload, state, false);
+
+                messages.add(msg);
+            }
+
+            return messages;
+        }
+    }
+
+    /** Returns the last N messages sent to this inbox */
+    public List<MqMessage> lastNMessages(String inboxName, int lastN) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT ID, RELATED_ID, FUNCTION, PAYLOAD, STATE, SENDER_INBOX FROM MESSAGE_QUEUE
+                     WHERE RECIPIENT_INBOX = ?
+                     ORDER BY ID DESC LIMIT ?
+                     """)) {
+
+            stmt.setString(1, inboxName);
+            stmt.setInt(2, lastN);
+            List<MqMessage> messages = new ArrayList<>(lastN);
+
+            var rs = stmt.executeQuery();
+            while (rs.next()) {
+                long msgId = rs.getLong(1);
+                long relatedId = rs.getLong(2);
+
+                String function = rs.getString(3);
+                String payload = rs.getString(4);
+
+                MqMessageState state = MqMessageState.valueOf(rs.getString(5));
+                boolean expectsResponse = rs.getBoolean(6);
+
+                var msg = new MqMessage(msgId, relatedId, function, payload, state, expectsResponse);
+
+                messages.add(msg);
+            }
+
+            // We want the last N messages in ascending order
+            return Lists.reverse(messages);
+        }
+
+    }
+
+    /** Modify the message indicated by id to have the given owner information */
+    public void changeOwner(long id, String instanceUUID, int tick) {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     UPDATE MESSAGE_QUEUE SET OWNER_INSTANCE=?, OWNER_TICK=?
+                     WHERE ID=?
+                     """)) {
+            stmt.setString(1, instanceUUID);
+            stmt.setInt(2, tick);
+            stmt.setLong(3, id);
+            stmt.executeUpdate();
+
+            if (!conn.getAutoCommit())
+                conn.commit();
+
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    /** Flags messages as dead if they have not been set to a terminal state within a TTL after the last update. */
+    public int reapDeadMessages() throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var setToDead = conn.prepareStatement("""
+                     UPDATE MESSAGE_QUEUE
+                     SET STATE='DEAD', UPDATED_TIME=CURRENT_TIMESTAMP(6)
+                     WHERE STATE IN ('NEW', 'ACK')
+                     AND TTL IS NOT NULL
+                     AND TIMESTAMPDIFF(SECOND, UPDATED_TIME, CURRENT_TIMESTAMP(6)) > TTL
+                     """)) {
+            int ret = setToDead.executeUpdate();
+            if (!conn.getAutoCommit())
+                conn.commit();
+            return ret;
+        }
+    }
+
+    /** Removes messages that have been set to a terminal state a while after their last update timestamp */
+    public int cleanOldMessages() throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var setToDead = conn.prepareStatement("""
+                     DELETE FROM MESSAGE_QUEUE
+                     WHERE STATE = 'OK'
+                     AND TTL IS NOT NULL
+                     AND TIMESTAMPDIFF(SECOND, UPDATED_TIME, CURRENT_TIMESTAMP(6)) > 3600
+                     """)) {
+            int ret = setToDead.executeUpdate();
+            if (!conn.getAutoCommit())
+                conn.commit();
+            return ret;
+        }
+    }
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/ActorStateMachine.java

@@ -0,0 +1,408 @@
+package nu.marginalia.mqsm;
+
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.MqMessage;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.inbox.MqInboxResponse;
+import nu.marginalia.mq.inbox.MqSubscription;
+import nu.marginalia.mq.inbox.MqSynchronousInbox;
+import nu.marginalia.mq.outbox.MqOutbox;
+import nu.marginalia.mqsm.graph.ResumeBehavior;
+import nu.marginalia.mqsm.graph.AbstractStateGraph;
+import nu.marginalia.mqsm.state.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BiConsumer;
+
+/** A state machine that can be used to implement an actor
+ * using a message queue as the persistence layer.  The state machine is
+ * resilient to crashes and can be resumed from the last state.
+ */
+public class ActorStateMachine {
+    private final Logger logger = LoggerFactory.getLogger(ActorStateMachine.class);
+
+    private final MqSynchronousInbox smInbox;
+    private final MqOutbox smOutbox;
+    private final String queueName;
+
+
+    private volatile MachineState state;
+    private volatile ExpectedMessage expectedMessage = ExpectedMessage.anyUnrelated();
+
+
+    private final MachineState errorState = new StateFactory.ErrorState();
+    private final MachineState finalState = new StateFactory.FinalState();
+    private final MachineState resumingState = new StateFactory.ResumingState();
+
+    private final List<BiConsumer<String, String>> stateChangeListeners = new ArrayList<>();
+    private final Map<String, MachineState> allStates = new HashMap<>();
+
+    private final boolean isDirectlyInitializable;
+
+    public ActorStateMachine(MessageQueueFactory messageQueueFactory,
+                             String queueName,
+                             UUID instanceUUID,
+                             AbstractStateGraph stateGraph)
+    {
+        this.queueName = queueName;
+
+        smInbox = messageQueueFactory.createSynchronousInbox(queueName, instanceUUID);
+        smOutbox = messageQueueFactory.createOutbox(queueName, queueName+"//out", instanceUUID);
+
+        smInbox.subscribe(new StateEventSubscription());
+
+        registerStates(List.of(errorState, finalState, resumingState));
+        registerStates(stateGraph);
+        isDirectlyInitializable = stateGraph.isDirectlyInitializable();
+
+        for (var declaredState : stateGraph.declaredStates()) {
+            if (!allStates.containsKey(declaredState.name())) {
+                throw new IllegalArgumentException("State " + declaredState.name() + " is not defined in the state graph");
+            }
+            if (!allStates.containsKey(declaredState.next())) {
+                throw new IllegalArgumentException("State " + declaredState.next() + " is not defined in the state graph");
+            }
+            for (var state : declaredState.transitions()) {
+                if (!allStates.containsKey(state)) {
+                    throw new IllegalArgumentException("State " + state + " is not defined in the state graph");
+                }
+            }
+        }
+
+        resume();
+
+        smInbox.start();
+    }
+
+    /** Listen to state changes */
+    public void listen(BiConsumer<String, String> listener) {
+        stateChangeListeners.add(listener);
+    }
+
+    /** Register the state graph */
+    void registerStates(List<MachineState> states) {
+        for (var state : states) {
+            allStates.put(state.name(), state);
+        }
+    }
+
+    /** Register the state graph */
+    void registerStates(AbstractStateGraph states) {
+        registerStates(states.asStateList());
+    }
+
+    /** Wait for the state machine to reach a final state.
+     * (possibly forever, halting problem and so on)
+     */
+    public void join() throws InterruptedException {
+        synchronized (this) {
+            if (null == state)
+                return;
+
+            while (!state.isFinal()) {
+                wait();
+            }
+        }
+    }
+
+    /** Wait for the state machine to reach a final state up to a given timeout.
+     */
+    public void join(long timeout, TimeUnit timeUnit) throws InterruptedException, TimeoutException {
+        long deadline = System.currentTimeMillis() + timeUnit.toMillis(timeout);
+
+        synchronized (this) {
+            if (null == state)
+                return;
+
+            while (!state.isFinal()) {
+                if (deadline <= System.currentTimeMillis())
+                    throw new TimeoutException("Timeout waiting for state machine to reach final state");
+                wait(100);
+            }
+        }
+    }
+
+    /** Initialize the state machine. */
+    public void init() throws Exception {
+        var transition = StateTransition.to("INITIAL");
+
+        synchronized (this) {
+            this.state = allStates.get(transition.state());
+            notifyAll();
+        }
+
+        smOutbox.sendNotice(transition.state(), transition.message());
+    }
+
+    /** Initialize the state machine. */
+    public void initFrom(String firstState) throws Exception {
+        var transition = StateTransition.to(firstState);
+
+        synchronized (this) {
+            this.state = allStates.get(transition.state());
+            notifyAll();
+        }
+
+        smOutbox.sendNotice(transition.state(), transition.message());
+    }
+
+    /** Initialize the state machine. */
+    public void init(String jsonEncodedArgument) throws Exception {
+        var transition = StateTransition.to("INITIAL", jsonEncodedArgument);
+
+        synchronized (this) {
+            this.state = allStates.get(transition.state());
+            notifyAll();
+        }
+
+        smOutbox.sendNotice(transition.state(), transition.message());
+    }
+
+    /** Initialize the state machine. */
+    public void initFrom(String state, String jsonEncodedArgument) throws Exception {
+        var transition = StateTransition.to(state, jsonEncodedArgument);
+
+        synchronized (this) {
+            this.state = allStates.get(transition.state());
+            notifyAll();
+        }
+
+        smOutbox.sendNotice(transition.state(), transition.message());
+    }
+
+    /** Resume the state machine from the last known state. */
+    private void resume() {
+
+        // We only permit resuming from the unitialized state
+        if (state != null) {
+            return;
+        }
+
+        // Fetch the last messages from the inbox
+        var message = smInbox.replay(5)
+                .stream()
+                .filter(m -> (m.state() == MqMessageState.NEW) || (m.state() == MqMessageState.ACK))
+                .findFirst();
+
+        if (message.isEmpty()) {
+            // No messages in the inbox, so start in a terminal state
+            expectedMessage = ExpectedMessage.anyUnrelated();
+            state = finalState;
+            return;
+        }
+
+        var firstMessage = message.get();
+        var resumeState = allStates.get(firstMessage.function());
+
+        logger.info("Resuming state machine from {}({})/{}", firstMessage.function(), firstMessage.payload(), firstMessage.state());
+        expectedMessage = ExpectedMessage.expectThis(firstMessage);
+
+        if (firstMessage.state() == MqMessageState.NEW) {
+            // The message is not acknowledged, so starting the inbox will trigger a state transition
+            // We still need to set a state here so that the join() method works
+
+            state = resumingState;
+        }
+        else if (firstMessage.state() == MqMessageState.ACK) {
+            resumeFromAck(resumeState, firstMessage);
+        }
+    }
+
+    private void resumeFromAck(MachineState resumeState,
+                               MqMessage message)
+    {
+        try {
+            if (resumeState.resumeBehavior().equals(ResumeBehavior.ERROR)) {
+                // The message is acknowledged, but the state does not support resuming
+                smOutbox.sendNotice(expectedMessage.id, "ERROR", "Illegal resumption from ACK'ed state " + message.function());
+            }
+            else if (resumeState.resumeBehavior().equals(ResumeBehavior.RESTART)) {
+                this.state = resumeState;
+
+                // The message is already acknowledged, we flag it as dead and then send an identical message
+                smOutbox.flagAsDead(message.msgId());
+                expectedMessage = ExpectedMessage.responseTo(message);
+                smOutbox.sendNotice(message.msgId(), "INITIAL", "");
+            }
+            else {
+                this.state = resumeState;
+
+                // The message is already acknowledged, we flag it as dead and then send an identical message
+                smOutbox.flagAsDead(message.msgId());
+                expectedMessage = ExpectedMessage.responseTo(message);
+                smOutbox.sendNotice(message.msgId(), message.function(), message.payload());
+            }
+        }
+        catch (Exception e) {
+            logger.error("Failed to replay state", e);
+        }
+    }
+
+    public void stop() throws InterruptedException {
+        smInbox.stop();
+        smOutbox.stop();
+    }
+
+    private void onStateTransition(MqMessage msg) {
+        final String nextState = msg.function();
+        final String data = msg.payload();
+
+        final long relatedId = msg.relatedId();
+
+        if (!expectedMessage.isExpected(msg)) {
+            // We've received a message that we didn't expect, throwing an exception will cause it to be flagged
+            // as an error in the message queue;  the message queue will proceed
+
+            throw new IllegalStateException("Unexpected message id " + relatedId + ", expected " + expectedMessage.id);
+        }
+
+        try {
+            logger.info("FSM State change in {}: {}->{}({})",
+                    queueName,
+                    state == null ? "[null]" : state.name(),
+                    nextState,
+                    data);
+
+            if (!allStates.containsKey(nextState)) {
+                logger.error("Unknown state {}", nextState);
+                setErrorState();
+                return;
+            }
+
+            synchronized (this) {
+                this.state = allStates.get(nextState);
+                notifyAll();
+            }
+
+            if (!state.isFinal()) {
+                logger.info("Transitining from state {}", state.name());
+                var transition = state.next(msg.payload());
+
+                if (!expectedMessage.isExpected(msg)) {
+                    logger.warn("Expected message changed during execution, skipping state transition to {}", transition.state());
+                }
+                else {
+                    expectedMessage = ExpectedMessage.responseTo(msg);
+                    smOutbox.sendNotice(expectedMessage.id, transition.state(), transition.message());
+                }
+            }
+            else {
+                // On terminal transition, we expect any message
+                expectedMessage = ExpectedMessage.anyUnrelated();
+            }
+        }
+        catch (Exception e) {
+            logger.error("Error in state machine transition", e);
+            setErrorState();
+        }
+    }
+
+    private void setErrorState() {
+        synchronized (this) {
+            state = errorState;
+            notifyAll();
+        }
+    }
+
+    public MachineState getState() {
+        return state;
+    }
+
+    public void abortExecution() throws Exception {
+        // Create a fake message to abort the execution
+        // This helps make sense of the queue when debugging
+        // and also permits the real termination message to have an
+        // unique expected ID
+
+        long abortMsgId = smOutbox.sendNotice(expectedMessage.id, "ABORT", "Aborting execution");
+
+        // Set it as dead to clean up the queue from mystery ACK messages
+        smOutbox.flagAsDead(abortMsgId);
+
+        // Set the expected message to the abort message,
+        // technically there's a slight chance of a race condition here,
+        // which will cause this message to be ERR'd and the process to
+        // continue, but it's very unlikely and the worst that can happen
+        // is you have to abort twice.
+
+        expectedMessage = ExpectedMessage.expectId(abortMsgId);
+
+        // Add a state transition to the final state
+        smOutbox.sendNotice(abortMsgId, finalState.name(), "");
+
+        // Dislodge the current task with an interrupt.
+        // It's actually fine if we accidentally interrupt the wrong thread
+        // (i.e. the abort task), since it shouldn't be doing anything interruptable
+        smInbox.abortCurrentTask();
+    }
+
+    /** Returns true if there is an INITIAL state that requires no parameters */
+    public boolean isDirectlyInitializable() {
+        return isDirectlyInitializable;
+    }
+
+    private class StateEventSubscription implements MqSubscription {
+
+        @Override
+        public boolean filter(MqMessage rawMessage) {
+            return true;
+        }
+
+        @Override
+        public MqInboxResponse onRequest(MqMessage msg) {
+            return null;
+        }
+
+        @Override
+        public void onNotification(MqMessage msg) {
+            onStateTransition(msg);
+            try {
+                stateChangeListeners.forEach(l -> l.accept(msg.function(), msg.payload()));
+            }
+            catch (Exception ex) {
+                // Rethrowing this will flag the message as an error in the message queue
+                throw new RuntimeException("Error in state change listener", ex);
+            }
+        }
+    }
+}
+
+/** ExpectedMessage guards against spurious state changes being triggered by old messages in the queue
+ *
+ * It contains the message id of the last message that was processed, and the messages sent by the state machine to
+ * itself via the message queue all have relatedId set to expectedMessageId.  If the state machine is unitialized or
+ * in a terminal state, it will accept messages with relatedIds that are equal to -1.
+ * */
+class ExpectedMessage {
+    public final long id;
+    public ExpectedMessage(long id) {
+        this.id = id;
+    }
+
+    public static ExpectedMessage expectThis(MqMessage message) {
+        return new ExpectedMessage(message.relatedId());
+    }
+
+    public static ExpectedMessage responseTo(MqMessage message) {
+        return new ExpectedMessage(message.msgId());
+    }
+
+    public static ExpectedMessage anyUnrelated() {
+        return new ExpectedMessage(-1);
+    }
+
+    public static ExpectedMessage expectId(long id) {
+        return new ExpectedMessage(id);
+    }
+
+    public boolean isExpected(MqMessage message) {
+        if (id < 0)
+            return true;
+
+        return id == message.relatedId();
+    }
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/StateFactory.java

@@ -0,0 +1,143 @@
+package nu.marginalia.mqsm;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.mqsm.graph.ResumeBehavior;
+import nu.marginalia.mqsm.state.MachineState;
+import nu.marginalia.mqsm.state.StateTransition;
+
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+@Singleton
+public class StateFactory {
+    private final Gson gson;
+
+    @Inject
+    public StateFactory(Gson gson) {
+        this.gson = gson;
+    }
+
+    public <T> MachineState create(String name, ResumeBehavior resumeBehavior, Class<T> param, Function<T, StateTransition> logic) {
+        return new MachineState() {
+            @Override
+            public String name() {
+                return name;
+            }
+
+            @Override
+            public StateTransition next(String message) {
+
+                if (message.isEmpty()) {
+                    return logic.apply(null);
+                }
+
+                try {
+                    var paramObj = gson.fromJson(message, param);
+                    return logic.apply(paramObj);
+                }
+                catch (JsonSyntaxException ex) {
+                    throw new IllegalArgumentException("Failed to parse '" + message +
+                                "' into a '" + param.getSimpleName() + "'", ex);
+                }
+            }
+
+            @Override
+            public ResumeBehavior resumeBehavior() {
+                return resumeBehavior;
+            }
+
+            @Override
+            public boolean isFinal() {
+                return false;
+            }
+        };
+    }
+
+    public MachineState create(String name, ResumeBehavior resumeBehavior, Supplier<StateTransition> logic) {
+        return new MachineState() {
+            @Override
+            public String name() {
+                return name;
+            }
+
+            @Override
+            public StateTransition next(String message) {
+                return logic.get();
+            }
+
+            @Override
+            public ResumeBehavior resumeBehavior() {
+                return resumeBehavior;
+            }
+
+            @Override
+            public boolean isFinal() {
+                return false;
+            }
+        };
+    }
+
+    public StateTransition transition(String state) {
+        return StateTransition.to(state);
+    }
+
+    public StateTransition transition(String state, Object message) {
+
+        if (null == message) {
+            return StateTransition.to(state);
+        }
+
+        return StateTransition.to(state, gson.toJson(message));
+    }
+
+    public static class ErrorState implements MachineState {
+        @Override
+        public String name() { return "ERROR"; }
+
+        @Override
+        public StateTransition next(String message) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ResumeBehavior resumeBehavior() { return ResumeBehavior.RETRY; }
+
+        @Override
+        public boolean isFinal() { return true; }
+    }
+
+    public static class FinalState implements MachineState {
+        @Override
+        public String name() { return "END"; }
+
+        @Override
+        public StateTransition next(String message) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ResumeBehavior resumeBehavior() { return ResumeBehavior.RETRY; }
+
+        @Override
+        public boolean isFinal() { return true; }
+    }
+
+    public static class ResumingState implements MachineState {
+        @Override
+        public String name() { return "RESUMING"; }
+
+        @Override
+        public StateTransition next(String message) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ResumeBehavior resumeBehavior() { return ResumeBehavior.RETRY; }
+
+        @Override
+        public boolean isFinal() { return false; }
+    }
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/AbstractStateGraph.java

@@ -0,0 +1,176 @@
+package nu.marginalia.mqsm.graph;
+
+import nu.marginalia.mqsm.state.MachineState;
+import nu.marginalia.mqsm.StateFactory;
+import nu.marginalia.mqsm.state.StateTransition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.*;
+
+public abstract class AbstractStateGraph {
+    private final StateFactory stateFactory;
+    private static final Logger logger = LoggerFactory.getLogger(AbstractStateGraph.class);
+
+    public AbstractStateGraph(StateFactory stateFactory) {
+        this.stateFactory = stateFactory;
+    }
+
+    public void transition(String state) {
+        throw new ControlFlowException(state, null);
+    }
+
+    public <T> void transition(String state, T payload) {
+        throw new ControlFlowException(state, payload);
+    }
+
+    public void error() {
+        throw new ControlFlowException("ERROR", "");
+    }
+
+
+    public <T> void error(T payload) {
+        throw new ControlFlowException("ERROR", payload);
+    }
+
+    public void error(Exception ex) {
+        throw new ControlFlowException("ERROR", ex.getClass().getSimpleName() + ":" + ex.getMessage());
+    }
+
+    /** Check whether there is an INITIAL state that can be directly initialized
+     * without declared parameters. */
+    public boolean isDirectlyInitializable() {
+        for (var method : getClass().getMethods()) {
+            var gs = method.getAnnotation(GraphState.class);
+            if (gs == null) {
+                continue;
+            }
+            if ("INITIAL".equals(gs.name()) && method.getParameterCount() == 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public Set<GraphState> declaredStates() {
+        Set<GraphState> ret = new HashSet<>();
+
+        for (var method : getClass().getMethods()) {
+            var gs = method.getAnnotation(GraphState.class);
+            if (gs != null) {
+                ret.add(gs);
+            }
+        }
+
+        return ret;
+    }
+
+
+    public Set<TerminalGraphState> terminalStates() {
+        Set<TerminalGraphState> ret = new HashSet<>();
+
+        for (var method : getClass().getMethods()) {
+            var gs = method.getAnnotation(TerminalGraphState.class);
+            if (gs != null) {
+                ret.add(gs);
+            }
+        }
+
+        return ret;
+    }
+
+    public List<MachineState> asStateList() {
+        List<MachineState> ret = new ArrayList<>();
+
+        for (var method : getClass().getMethods()) {
+            var gs = method.getAnnotation(GraphState.class);
+            if (gs != null) {
+                ret.add(graphState(method, gs));
+            }
+
+            var ts = method.getAnnotation(TerminalGraphState.class);
+            if (ts != null) {
+                ret.add(stateFactory.create(ts.name(), ResumeBehavior.ERROR, () -> {
+                    throw new ControlFlowException(ts.name(), null);
+                }));
+            }
+        }
+
+        return ret;
+    }
+
+    private MachineState graphState(Method method, GraphState gs) {
+
+        var parameters =  method.getParameterTypes();
+        boolean returnsVoid = method.getGenericReturnType().equals(Void.TYPE);
+
+        if (parameters.length == 0) {
+            return stateFactory.create(gs.name(), gs.resume(), () -> {
+                try {
+                    if (returnsVoid) {
+                        method.invoke(this);
+                        return StateTransition.to(gs.next());
+                    } else {
+                        Object ret = method.invoke(this);
+                        return stateFactory.transition(gs.next(), ret);
+                    }
+                }
+                catch (Exception e) {
+                    return invocationExceptionToStateTransition(gs.name(), e);
+                }
+            });
+        }
+        else if (parameters.length == 1) {
+            return stateFactory.create(gs.name(), gs.resume(), parameters[0], (param) -> {
+                try {
+                    if (returnsVoid) {
+                        method.invoke(this,  param);
+                        return StateTransition.to(gs.next());
+                    } else {
+                        Object ret = method.invoke(this, param);
+                        return stateFactory.transition(gs.next(), ret);
+                    }
+                } catch (Exception e) {
+                    return invocationExceptionToStateTransition(gs.name(), e);
+                }
+            });
+        }
+        else {
+            // We permit only @GraphState-annotated methods like this:
+            //
+            // void foo();
+            // void foo(Object bar);
+            // Object foo();
+            // Object foo(Object bar);
+
+            throw new IllegalStateException("StateGraph " +
+                    getClass().getSimpleName() +
+                    " has invalid method signature for method " +
+                    method.getName() +
+                    ": Expected 0 or 1 parameter(s) but found " +
+                    Arrays.toString(parameters));
+        }
+    }
+
+    private StateTransition invocationExceptionToStateTransition(String state, Throwable ex) {
+        while (ex instanceof InvocationTargetException e) {
+            if (e.getCause() != null) ex = ex.getCause();
+        }
+
+        if (ex instanceof ControlFlowException cfe) {
+            return stateFactory.transition(cfe.getState(), cfe.getPayload());
+        }
+        else if (ex instanceof InterruptedException intE) {
+            logger.error("State execution was interrupted " + state);
+            return StateTransition.to("ERR", "Execution interrupted");
+        }
+        else {
+            logger.error("Error in state invocation " + state, ex);
+            return StateTransition.to("ERROR",
+                    "Exception: " + ex.getClass().getSimpleName() + "/" +  ex.getMessage());
+        }
+    }
+
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/ControlFlowException.java

@@ -0,0 +1,22 @@
+package nu.marginalia.mqsm.graph;
+
+/** Exception thrown by a state to indicate that the state machine should jump to a different state. */
+public class ControlFlowException extends RuntimeException {
+    private final String state;
+    private final Object payload;
+
+    public ControlFlowException(String state, Object payload) {
+        this.state = state;
+        this.payload = payload;
+    }
+
+    public String getState() {
+        return state;
+    }
+
+    public Object getPayload() {
+        return payload;
+    }
+
+    public StackTraceElement[] getStackTrace() { return new StackTraceElement[0]; }
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/GraphState.java

@@ -0,0 +1,15 @@
+package nu.marginalia.mqsm.graph;
+
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/** Annotation for declaring a state in an actor's state graph. */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface GraphState {
+    String name();
+    String next() default "ERROR";
+    String[] transitions() default {};
+    String description() default "";
+    ResumeBehavior resume() default ResumeBehavior.ERROR;
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/ResumeBehavior.java

@@ -0,0 +1,10 @@
+package nu.marginalia.mqsm.graph;
+
+public enum ResumeBehavior {
+    /** Retry the state on resume */
+    RETRY,
+    /** Jump to ERROR on resume if the message has been acknowledged */
+    ERROR,
+    /** Jump to INITIAL on resume */
+    RESTART
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/TerminalGraphState.java

@@ -0,0 +1,10 @@
+package nu.marginalia.mqsm.graph;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Retention(RetentionPolicy.RUNTIME)
+public @interface TerminalGraphState {
+    String name();
+    String description() default "";
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/state/MachineState.java

@@ -0,0 +1,14 @@
+package nu.marginalia.mqsm.state;
+
+import nu.marginalia.mqsm.graph.ResumeBehavior;
+
+public interface MachineState {
+    String name();
+
+    StateTransition next(String message);
+
+    ResumeBehavior resumeBehavior();
+
+    boolean isFinal();
+
+}

code/common/message-queue/src/main/java/nu/marginalia/mqsm/state/StateTransition.java

@@ -0,0 +1,11 @@
+package nu.marginalia.mqsm.state;
+
+public record StateTransition(String state, String message) {
+    public static StateTransition to(String state) {
+        return new StateTransition(state, "");
+    }
+
+    public static StateTransition to(String state, String message) {
+        return new StateTransition(state, message);
+    }
+}

code/common/message-queue/src/test/java/nu/marginalia/mq/MqMessageRow.java

@@ -0,0 +1,21 @@
+package nu.marginalia.mq;
+
+import nu.marginalia.mq.MqMessageState;
+
+import javax.annotation.Nullable;
+
+public record MqMessageRow (
+        long id,
+        long relatedId,
+        @Nullable
+        String senderInbox,
+        String recipientInbox,
+        String function,
+        String payload,
+        MqMessageState state,
+        String ownerInstance,
+        long ownerTick,
+        long createdTime,
+        long updatedTime,
+        long ttl
+) {}

code/common/message-queue/src/test/java/nu/marginalia/mq/MqTestUtil.java

@@ -0,0 +1,51 @@
+package nu.marginalia.mq;
+
+import com.zaxxer.hikari.HikariDataSource;
+import org.junit.jupiter.api.Assertions;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class MqTestUtil {
+    public static List<MqMessageRow> getMessages(HikariDataSource dataSource, String inbox) {
+        List<MqMessageRow> messages = new ArrayList<>();
+
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                SELECT ID, RELATED_ID,
+                        SENDER_INBOX, RECIPIENT_INBOX,
+                        FUNCTION, PAYLOAD,
+                        STATE,
+                        OWNER_INSTANCE, OWNER_TICK,
+                        CREATED_TIME, UPDATED_TIME,
+                        TTL
+                FROM MESSAGE_QUEUE
+                WHERE RECIPIENT_INBOX = ?
+                 """))
+        {
+            stmt.setString(1, inbox);
+            var rsp = stmt.executeQuery();
+            while (rsp.next()) {
+                messages.add(new MqMessageRow(
+                        rsp.getLong("ID"),
+                        rsp.getLong("RELATED_ID"),
+                        rsp.getString("SENDER_INBOX"),
+                        rsp.getString("RECIPIENT_INBOX"),
+                        rsp.getString("FUNCTION"),
+                        rsp.getString("PAYLOAD"),
+                        MqMessageState.valueOf(rsp.getString("STATE")),
+                        rsp.getString("OWNER_INSTANCE"),
+                        rsp.getLong("OWNER_TICK"),
+                        rsp.getTimestamp("CREATED_TIME").getTime(),
+                        rsp.getTimestamp("UPDATED_TIME").getTime(),
+                        rsp.getLong("TTL")
+                ));
+            }
+        }
+        catch (SQLException ex) {
+            Assertions.fail(ex);
+        }
+        return messages;
+    }
+}

code/common/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java

@@ -0,0 +1,308 @@
+package nu.marginalia.mq.outbox;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.mq.MqMessage;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.MqTestUtil;
+import nu.marginalia.mq.inbox.*;
+import nu.marginalia.mq.persistence.MqPersistence;
+import org.junit.jupiter.api.*;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+@Tag("slow")
+@Testcontainers
+public class MqOutboxTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    private String inboxId;
+
+    @BeforeEach
+    public void setUp() {
+        inboxId = UUID.randomUUID().toString();
+    }
+    @BeforeAll
+    public static void setUpAll() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        dataSource.close();
+    }
+
+    @Test
+    public void testOpenClose() throws InterruptedException {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, inboxId+"/reply", UUID.randomUUID());
+        outbox.stop();
+    }
+
+    @Test
+    public void testSingleShotInboxTimeout() throws Exception {
+        var inbox = new MqSingleShotInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+        var message = inbox.waitForMessage(100, TimeUnit.MILLISECONDS);
+        assertTrue(message.isEmpty());
+    }
+
+    @Test
+    public void testOutboxTimeout() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId, inboxId+"/reply", UUID.randomUUID());
+        long id = outbox.sendAsync("test", "Hello World");
+        try {
+            outbox.waitResponse(id, 100, TimeUnit.MILLISECONDS);
+        }
+        catch (TimeoutException ex) {
+            return; // ok
+        }
+        catch (Exception ex) {
+            ex.printStackTrace();
+        }
+        fail();
+    }
+
+    @Test
+    public void testSingleShotInbox() throws Exception {
+        // Send a message to the inbox
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+        long id = outbox.sendAsync("test", "Hello World");
+
+        // Create a single-shot inbox
+        var inbox = new MqSingleShotInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+
+        // Wait for the message to arrive
+        var message = inbox.waitForMessage(1, TimeUnit.SECONDS);
+
+        // Check that the message arrived
+        assertTrue(message.isPresent());
+        assertEquals("Hello World", message.get().payload());
+
+        // Send a response
+        inbox.sendResponse(message.get(), new MqInboxResponse("Alright then", MqMessageState.OK));
+
+        // Wait for the response to arrive
+        var response = outbox.waitResponse(id, 1, TimeUnit.SECONDS);
+
+        // Check that the response arrived
+        assertEquals(MqMessageState.OK, response.state());
+        assertEquals("Alright then", response.payload());
+    }
+
+    @Test
+    public void testSend() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+        Executors.newSingleThreadExecutor().submit(() -> outbox.send("test", "Hello World"));
+
+        TimeUnit.MILLISECONDS.sleep(100);
+
+        var messages = MqTestUtil.getMessages(dataSource, inboxId);
+        assertEquals(1, messages.size());
+        System.out.println(messages.get(0));
+
+        outbox.stop();
+    }
+
+
+    @Test
+    public void testSendAndRespondAsyncInbox() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+
+        var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+        inbox.subscribe(justRespond("Alright then"));
+        inbox.start();
+
+        var rsp = outbox.send("test", "Hello World");
+
+        assertEquals(MqMessageState.OK, rsp.state());
+        assertEquals("Alright then", rsp.payload());
+
+        var messages = MqTestUtil.getMessages(dataSource, inboxId);
+        assertEquals(1, messages.size());
+        assertEquals(MqMessageState.OK, messages.get(0).state());
+
+        outbox.stop();
+        inbox.stop();
+    }
+
+    @Test
+    public void testSendAndRespondSyncInbox() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+
+        var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+        inbox.subscribe(justRespond("Alright then"));
+        inbox.start();
+
+        var rsp = outbox.send("test", "Hello World");
+
+        assertEquals(MqMessageState.OK, rsp.state());
+        assertEquals("Alright then", rsp.payload());
+
+        var messages = MqTestUtil.getMessages(dataSource, inboxId);
+        assertEquals(1, messages.size());
+        assertEquals(MqMessageState.OK, messages.get(0).state());
+
+        outbox.stop();
+        inbox.stop();
+    }
+
+    @Test
+    public void testSendMultipleAsyncInbox() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+
+        var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+        inbox.subscribe(echo());
+        inbox.start();
+
+        var rsp1 = outbox.send("test", "one");
+        var rsp2 = outbox.send("test", "two");
+        var rsp3 = outbox.send("test", "three");
+        var rsp4 = outbox.send("test", "four");
+
+        Thread.sleep(500);
+
+        assertEquals(MqMessageState.OK, rsp1.state());
+        assertEquals("one", rsp1.payload());
+        assertEquals(MqMessageState.OK, rsp2.state());
+        assertEquals("two", rsp2.payload());
+        assertEquals(MqMessageState.OK, rsp3.state());
+        assertEquals("three", rsp3.payload());
+        assertEquals(MqMessageState.OK, rsp4.state());
+        assertEquals("four", rsp4.payload());
+
+        var messages = MqTestUtil.getMessages(dataSource, inboxId);
+        assertEquals(4, messages.size());
+        for (var message : messages) {
+            assertEquals(MqMessageState.OK, message.state());
+        }
+
+        outbox.stop();
+        inbox.stop();
+    }
+
+    @Test
+    public void testSendMultipleSyncInbox() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+
+        var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+        inbox.subscribe(echo());
+        inbox.start();
+
+        var rsp1 = outbox.send("test", "one");
+        var rsp2 = outbox.send("test", "two");
+        var rsp3 = outbox.send("test", "three");
+        var rsp4 = outbox.send("test", "four");
+
+        Thread.sleep(500);
+
+        assertEquals(MqMessageState.OK, rsp1.state());
+        assertEquals("one", rsp1.payload());
+        assertEquals(MqMessageState.OK, rsp2.state());
+        assertEquals("two", rsp2.payload());
+        assertEquals(MqMessageState.OK, rsp3.state());
+        assertEquals("three", rsp3.payload());
+        assertEquals(MqMessageState.OK, rsp4.state());
+        assertEquals("four", rsp4.payload());
+
+        var messages = MqTestUtil.getMessages(dataSource, inboxId);
+        assertEquals(4, messages.size());
+        for (var message : messages) {
+            assertEquals(MqMessageState.OK, message.state());
+        }
+
+        outbox.stop();
+        inbox.stop();
+    }
+
+    @Test
+    public void testSendAndRespondWithErrorHandlerAsyncInbox() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+        var inbox = new MqAsynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+
+        inbox.start();
+
+        var rsp = outbox.send("test", "Hello World");
+
+        assertEquals(MqMessageState.ERR, rsp.state());
+
+        var messages = MqTestUtil.getMessages(dataSource, inboxId);
+        assertEquals(1, messages.size());
+        assertEquals(MqMessageState.ERR, messages.get(0).state());
+
+        outbox.stop();
+        inbox.stop();
+    }
+
+    @Test
+    public void testSendAndRespondWithErrorHandlerSyncInbox() throws Exception {
+        var outbox = new MqOutbox(new MqPersistence(dataSource), inboxId,inboxId+"/reply", UUID.randomUUID());
+        var inbox = new MqSynchronousInbox(new MqPersistence(dataSource), inboxId, UUID.randomUUID());
+
+        inbox.start();
+
+        var rsp = outbox.send("test", "Hello World");
+
+        assertEquals(MqMessageState.ERR, rsp.state());
+
+        var messages = MqTestUtil.getMessages(dataSource, inboxId);
+        assertEquals(1, messages.size());
+        assertEquals(MqMessageState.ERR, messages.get(0).state());
+
+        outbox.stop();
+        inbox.stop();
+    }
+
+    public MqSubscription justRespond(String response) {
+        return new MqSubscription() {
+            @Override
+            public boolean filter(MqMessage rawMessage) {
+                return true;
+            }
+
+            @Override
+            public MqInboxResponse onRequest(MqMessage msg) {
+                return MqInboxResponse.ok(response);
+            }
+
+            @Override
+            public void onNotification(MqMessage msg) { }
+        };
+    }
+
+    public MqSubscription echo() {
+        return new MqSubscription() {
+            @Override
+            public boolean filter(MqMessage rawMessage) {
+                return true;
+            }
+
+            @Override
+            public MqInboxResponse onRequest(MqMessage msg) {
+                return MqInboxResponse.ok(msg.payload());
+            }
+
+            @Override
+            public void onNotification(MqMessage msg) {}
+        };
+    }
+
+}

code/common/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java

@@ -0,0 +1,190 @@
+package nu.marginalia.mq.persistence;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.MqTestUtil;
+import org.junit.jupiter.api.*;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+@Tag("slow")
+@Testcontainers
+public class MqPersistenceTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    static MqPersistence persistence;
+    String recipientId;
+    String senderId;
+
+    @BeforeEach
+    public void setUp() {
+        senderId = UUID.randomUUID().toString();
+        recipientId = UUID.randomUUID().toString();
+    }
+
+    @BeforeAll
+    public static void setUpAll() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+        persistence = new MqPersistence(dataSource);
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        dataSource.close();
+    }
+
+    @Test
+    public void testReaper() throws Exception {
+
+        long id = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(2));
+        persistence.reapDeadMessages();
+
+        var messages = MqTestUtil.getMessages(dataSource, recipientId);
+        assertEquals(1, messages.size());
+        assertEquals(MqMessageState.NEW, messages.get(0).state());
+        System.out.println(messages);
+
+        TimeUnit.SECONDS.sleep(5);
+
+        persistence.reapDeadMessages();
+
+        messages = MqTestUtil.getMessages(dataSource, recipientId);
+        assertEquals(1, messages.size());
+        assertEquals(MqMessageState.DEAD, messages.get(0).state());
+    }
+
+    @Test
+    public void sendWithReplyAddress() throws Exception {
+
+        long id = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(30));
+
+        var messages = MqTestUtil.getMessages(dataSource, recipientId);
+        assertEquals(1, messages.size());
+
+        var message = messages.get(0);
+
+        assertEquals(id, message.id());
+        assertEquals("function", message.function());
+        assertEquals("payload", message.payload());
+        assertEquals(MqMessageState.NEW, message.state());
+
+        System.out.println(message);
+    }
+
+    @Test
+    public void sendNoReplyAddress() throws Exception {
+
+        long id = persistence.sendNewMessage(recipientId, null, null, "function", "payload", Duration.ofSeconds(30));
+
+        var messages = MqTestUtil.getMessages(dataSource, recipientId);
+        assertEquals(1, messages.size());
+
+        var message = messages.get(0);
+
+        assertEquals(id, message.id());
+        assertNull(message.senderInbox());
+        assertEquals("function", message.function());
+        assertEquals("payload", message.payload());
+        assertEquals(MqMessageState.NEW, message.state());
+
+        System.out.println(message);
+    }
+
+    @Test
+    public void updateState() throws Exception {
+
+        long id = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(30));
+        persistence.updateMessageState(id, MqMessageState.OK);
+        System.out.println(id);
+
+        var messages = MqTestUtil.getMessages(dataSource, recipientId);
+        assertEquals(1, messages.size());
+
+        var message = messages.get(0);
+
+        assertEquals(id, message.id());
+        assertEquals(MqMessageState.OK, message.state());
+
+        System.out.println(message);
+    }
+
+    @Test
+    public void testReply() throws Exception {
+        long request = persistence.sendNewMessage(recipientId, senderId, null, "function", "payload", Duration.ofSeconds(30));
+        long response = persistence.sendResponse(request,  MqMessageState.OK, "response");
+
+        var sentMessages = MqTestUtil.getMessages(dataSource, recipientId);
+        System.out.println(sentMessages);
+        assertEquals(1, sentMessages.size());
+
+        var requestMessage = sentMessages.get(0);
+        assertEquals(request, requestMessage.id());
+        assertEquals(MqMessageState.OK, requestMessage.state());
+
+
+        var replies = MqTestUtil.getMessages(dataSource, senderId);
+        System.out.println(replies);
+        assertEquals(1, replies.size());
+
+        var responseMessage = replies.get(0);
+        assertEquals(response, responseMessage.id());
+        assertEquals(request, responseMessage.relatedId());
+        assertEquals(MqMessageState.NEW, responseMessage.state());
+    }
+
+    @Test
+    public void testPollInbox() throws Exception {
+
+        String instanceId = "BATMAN";
+        long tick = 1234L;
+
+        long id = persistence.sendNewMessage(recipientId,  null, null, "function", "payload", Duration.ofSeconds(30));
+
+        var messagesPollFirstTime = persistence.pollInbox(recipientId, instanceId , tick, 10);
+
+        /** CHECK POLL RESULT */
+        assertEquals(1, messagesPollFirstTime.size());
+        var firstPollMessage = messagesPollFirstTime.iterator().next();
+        assertEquals(id, firstPollMessage.msgId());
+        assertEquals("function", firstPollMessage.function());
+        assertEquals("payload", firstPollMessage.payload());
+
+        /** CHECK DB TABLE */
+        var messages = MqTestUtil.getMessages(dataSource, recipientId);
+        assertEquals(1, messages.size());
+
+        var message = messages.get(0);
+
+        assertEquals(id, message.id());
+        assertEquals("function", message.function());
+        assertEquals("payload", message.payload());
+        assertEquals(MqMessageState.ACK, message.state());
+        assertEquals(instanceId, message.ownerInstance());
+        assertEquals(tick, message.ownerTick());
+
+        /** VERIFY SECOND POLL IS EMPTY */
+        var messagePollSecondTime = persistence.pollInbox(recipientId, instanceId , 1, 10);
+        assertEquals(0, messagePollSecondTime.size());
+    }
+}

code/common/message-queue/src/test/java/nu/marginalia/mqsm/ActorStateMachineErrorTest.java

@@ -0,0 +1,104 @@
+package nu.marginalia.mqsm;
+
+import com.google.gson.GsonBuilder;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.MqMessageRow;
+import nu.marginalia.mq.MqTestUtil;
+import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.mqsm.graph.GraphState;
+import nu.marginalia.mqsm.graph.AbstractStateGraph;
+import nu.marginalia.mqsm.graph.ResumeBehavior;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.parallel.Execution;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+@Tag("slow")
+@Testcontainers
+@Execution(SAME_THREAD)
+public class ActorStateMachineErrorTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    static MqPersistence persistence;
+    static MessageQueueFactory messageQueueFactory;
+    private String inboxId;
+
+    @BeforeEach
+    public void setUp() {
+        inboxId = UUID.randomUUID().toString();
+    }
+    @BeforeAll
+    public static void setUpAll() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+        persistence = new MqPersistence(dataSource);
+        messageQueueFactory = new MessageQueueFactory(persistence);
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        dataSource.close();
+    }
+
+    public static class ErrorHurdles extends AbstractStateGraph {
+
+        public ErrorHurdles(StateFactory stateFactory) {
+            super(stateFactory);
+        }
+
+        @GraphState(name = "INITIAL", next = "FAILING")
+        public void initial() {
+
+        }
+        @GraphState(name = "FAILING", next = "OK", resume = ResumeBehavior.RETRY)
+        public void resumable() {
+            throw new RuntimeException("Boom!");
+        }
+        @GraphState(name = "OK", next = "END")
+        public void ok() {
+
+        }
+
+    }
+
+    @Test
+    public void smResumeResumableFromNew() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ErrorHurdles(stateFactory));
+
+        sm.init();
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        List<String> states = MqTestUtil.getMessages(dataSource, inboxId)
+                .stream()
+                .peek(System.out::println)
+                .map(MqMessageRow::function)
+                .toList();
+
+        assertEquals(List.of("INITIAL", "FAILING", "ERROR"), states);
+    }
+
+}

code/common/message-queue/src/test/java/nu/marginalia/mqsm/ActorStateMachineNullTest.java

@@ -0,0 +1,98 @@
+package nu.marginalia.mqsm;
+
+import com.google.gson.GsonBuilder;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.MqTestUtil;
+import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.mqsm.graph.AbstractStateGraph;
+import nu.marginalia.mqsm.graph.GraphState;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.parallel.Execution;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+@Tag("slow")
+@Testcontainers
+@Execution(SAME_THREAD)
+public class ActorStateMachineNullTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    static MqPersistence persistence;
+    static MessageQueueFactory messageQueueFactory;
+    private String inboxId;
+
+    @BeforeEach
+    public void setUp() {
+        inboxId = UUID.randomUUID().toString();
+    }
+    @BeforeAll
+    public static void setUpAll() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+        persistence = new MqPersistence(dataSource);
+        messageQueueFactory = new MessageQueueFactory(persistence);
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        dataSource.close();
+    }
+
+    public static class TestGraph extends AbstractStateGraph {
+        public TestGraph(StateFactory stateFactory) {
+            super(stateFactory);
+        }
+
+        @GraphState(name = "INITIAL", next = "GREET")
+        public void initial() {}
+
+        @GraphState(name = "GREET", next = "END")
+        public void greet(String message) {
+            if (null == message) {
+                System.out.println("Hello, null!");
+                return;
+            }
+            Assertions.fail("Should not be called");
+        }
+
+    }
+
+    @Test
+    public void testStateGraphNullSerialization() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+        var graph = new TestGraph(stateFactory);
+
+
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), graph);
+        sm.registerStates(graph);
+
+        sm.init();
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println);
+
+    }
+
+}

code/common/message-queue/src/test/java/nu/marginalia/mqsm/ActorStateMachineResumeTest.java

@@ -0,0 +1,186 @@
+package nu.marginalia.mqsm;
+
+import com.google.gson.GsonBuilder;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.MqMessageRow;
+import nu.marginalia.mq.MqMessageState;
+import nu.marginalia.mq.MqTestUtil;
+import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.mqsm.graph.GraphState;
+import nu.marginalia.mqsm.graph.AbstractStateGraph;
+import nu.marginalia.mqsm.graph.ResumeBehavior;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.parallel.Execution;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+@Tag("slow")
+@Testcontainers
+@Execution(SAME_THREAD)
+public class ActorStateMachineResumeTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    static MqPersistence persistence;
+    static MessageQueueFactory messageQueueFactory;
+    private String inboxId;
+
+    @BeforeEach
+    public void setUp() {
+        inboxId = UUID.randomUUID().toString();
+    }
+    @BeforeAll
+    public static void setUpAll() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+        persistence = new MqPersistence(dataSource);
+        messageQueueFactory = new MessageQueueFactory(persistence);
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        dataSource.close();
+    }
+
+    public static class ResumeTrialsGraph extends AbstractStateGraph {
+
+        public ResumeTrialsGraph(StateFactory stateFactory) {
+            super(stateFactory);
+        }
+
+        @GraphState(name = "INITIAL", next = "RESUMABLE")
+        public void initial() {}
+        @GraphState(name = "RESUMABLE", next = "NON-RESUMABLE", resume = ResumeBehavior.RETRY)
+        public void resumable() {}
+        @GraphState(name = "NON-RESUMABLE", next = "OK", resume = ResumeBehavior.ERROR)
+        public void nonResumable() {}
+
+        @GraphState(name = "OK", next = "END")
+        public void ok() {}
+
+    }
+
+    @Test
+    public void smResumeResumableFromNew() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+
+
+        persistence.sendNewMessage(inboxId,  null, -1L, "RESUMABLE", "", null);
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsGraph(stateFactory));
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        List<String> states = MqTestUtil.getMessages(dataSource, inboxId)
+                .stream()
+                .peek(System.out::println)
+                .map(MqMessageRow::function)
+                .toList();
+
+        assertEquals(List.of("RESUMABLE", "NON-RESUMABLE", "OK", "END"), states);
+    }
+
+    @Test
+    public void smResumeFromAck() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+
+        long id = persistence.sendNewMessage(inboxId,  null, -1L, "RESUMABLE", "", null);
+        persistence.updateMessageState(id, MqMessageState.ACK);
+
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsGraph(stateFactory));
+
+        sm.join(4, TimeUnit.SECONDS);
+        sm.stop();
+
+        List<String> states = MqTestUtil.getMessages(dataSource, inboxId)
+                .stream()
+                .peek(System.out::println)
+                .map(MqMessageRow::function)
+                .toList();
+
+        assertEquals(List.of("RESUMABLE", "RESUMABLE", "NON-RESUMABLE", "OK", "END"), states);
+    }
+
+
+    @Test
+    public void smResumeNonResumableFromNew() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+
+
+        persistence.sendNewMessage(inboxId,  null, -1L, "NON-RESUMABLE", "", null);
+
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsGraph(stateFactory));
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        List<String> states = MqTestUtil.getMessages(dataSource, inboxId)
+                .stream()
+                .peek(System.out::println)
+                .map(MqMessageRow::function)
+                .toList();
+
+        assertEquals(List.of("NON-RESUMABLE", "OK", "END"), states);
+    }
+
+    @Test
+    public void smResumeNonResumableFromAck() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+
+
+        long id = persistence.sendNewMessage(inboxId,  null, null, "NON-RESUMABLE", "", null);
+        persistence.updateMessageState(id, MqMessageState.ACK);
+
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsGraph(stateFactory));
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        List<String> states = MqTestUtil.getMessages(dataSource, inboxId)
+                .stream()
+                .peek(System.out::println)
+                .map(MqMessageRow::function)
+                .toList();
+
+        assertEquals(List.of("NON-RESUMABLE", "ERROR"), states);
+    }
+
+    @Test
+    public void smResumeEmptyQueue() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+
+
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new ResumeTrialsGraph(stateFactory));
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        List<String> states = MqTestUtil.getMessages(dataSource, inboxId)
+                .stream()
+                .peek(System.out::println)
+                .map(MqMessageRow::function)
+                .toList();
+
+        assertEquals(List.of(), states);
+    }
+}

code/common/message-queue/src/test/java/nu/marginalia/mqsm/ActorStateMachineTest.java

@@ -0,0 +1,144 @@
+package nu.marginalia.mqsm;
+
+import com.google.gson.GsonBuilder;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.mq.MessageQueueFactory;
+import nu.marginalia.mq.MqTestUtil;
+import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.mqsm.graph.GraphState;
+import nu.marginalia.mqsm.graph.AbstractStateGraph;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.parallel.Execution;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+@Tag("slow")
+@Testcontainers
+@Execution(SAME_THREAD)
+public class ActorStateMachineTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    static MqPersistence persistence;
+    static MessageQueueFactory messageQueueFactory;
+    private String inboxId;
+
+    @BeforeEach
+    public void setUp() {
+        inboxId = UUID.randomUUID().toString();
+    }
+    @BeforeAll
+    public static void setUpAll() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+        persistence = new MqPersistence(dataSource);
+        messageQueueFactory = new MessageQueueFactory(persistence);
+    }
+
+    @AfterAll
+    public static void tearDownAll() {
+        dataSource.close();
+    }
+
+    public static class TestGraph extends AbstractStateGraph {
+        public TestGraph(StateFactory stateFactory) {
+            super(stateFactory);
+        }
+
+        @GraphState(name = "INITIAL", next = "GREET")
+        public String initial() {
+            return "World";
+        }
+
+        @GraphState(name = "GREET")
+        public void greet(String message) {
+            System.out.println("Hello, " + message + "!");
+
+            transition("COUNT-DOWN", 5);
+        }
+
+        @GraphState(name = "COUNT-DOWN", next = "END")
+        public void countDown(Integer from) {
+            if (from > 0) {
+                System.out.println(from);
+                transition("COUNT-DOWN", from - 1);
+            }
+        }
+    }
+
+    @Test
+    public void testAnnotatedStateGraph() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+        var graph = new TestGraph(stateFactory);
+
+
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), graph);
+        sm.registerStates(graph);
+
+        sm.init();
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println);
+
+    }
+
+    @Test
+    public void testStartStopStartStop() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new TestGraph(stateFactory));
+
+        sm.init();
+
+        Thread.sleep(150);
+        sm.stop();
+
+        System.out.println("-------------------- ");
+
+        var sm2 = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new TestGraph(stateFactory));
+        sm2.join(2, TimeUnit.SECONDS);
+        sm2.stop();
+
+        MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println);
+    }
+
+    @Test
+    public void testFalseTransition() throws Exception {
+        var stateFactory = new StateFactory(new GsonBuilder().create());
+
+        // Prep the queue with a message to set the state to initial,
+        // and an additional message to trigger the false transition back to initial
+
+        persistence.sendNewMessage(inboxId,  null, null, "INITIAL", "", null);
+        persistence.sendNewMessage(inboxId,  null, null, "INITIAL", "", null);
+
+        var sm = new ActorStateMachine(messageQueueFactory, inboxId, UUID.randomUUID(), new TestGraph(stateFactory));
+
+        Thread.sleep(50);
+
+        sm.join(2, TimeUnit.SECONDS);
+        sm.stop();
+
+        MqTestUtil.getMessages(dataSource, inboxId).forEach(System.out::println);
+    }
+
+}

code/common/model/src/main/java/nu/marginalia/model/EdgeDomain.java

@@ -3,13 +3,15 @@ package nu.marginalia.model;
 import lombok.*;
 
 import javax.annotation.Nonnull;
+import java.io.Serializable;
 import java.util.Objects;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
 @AllArgsConstructor
 @Getter @Setter @Builder
-public class EdgeDomain {
+public class EdgeDomain implements Serializable {
+
     @Nonnull
     public final String subDomain;
     @Nonnull
@@ -160,22 +162,16 @@ public class EdgeDomain {
 
     public boolean equals(final Object o) {
         if (o == this) return true;
-        if (!(o instanceof EdgeDomain)) return false;
-        final EdgeDomain other = (EdgeDomain) o;
-        if (!other.canEqual((Object) this)) return false;
+        if (!(o instanceof EdgeDomain other)) return false;
         final String this$subDomain = this.getSubDomain();
         final String other$subDomain = other.getSubDomain();
-        if (!this$subDomain.equalsIgnoreCase(other$subDomain)) return false;
+        if (!Objects.equals(this$subDomain,other$subDomain)) return false;
         final String this$domain = this.getDomain();
         final String other$domain = other.getDomain();
-        if (!this$domain.equalsIgnoreCase(other$domain)) return false;
+        if (!Objects.equals(this$domain,other$domain)) return false;
         return true;
     }
 
-    protected boolean canEqual(final Object other) {
-        return other instanceof EdgeDomain;
-    }
-
     public int hashCode() {
         final int PRIME = 59;
         int result = 1;

code/common/model/src/main/java/nu/marginalia/model/EdgeUrl.java

@@ -5,6 +5,8 @@ import lombok.Getter;
 import lombok.Setter;
 import nu.marginalia.util.QueryParams;
 
+import javax.annotation.Nullable;
+import java.io.Serializable;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -14,7 +16,7 @@ import java.util.Optional;
 import java.util.regex.Pattern;
 
 @Getter @Setter @Builder
-public class EdgeUrl {
+public class EdgeUrl implements Serializable {
     public final String proto;
     public final EdgeDomain domain;
     public final Integer port;
@@ -33,8 +35,12 @@ public class EdgeUrl {
         this(new URI(urlencodeFixer(url)));
     }
 
-    public static Optional<EdgeUrl> parse(String url) {
+    public static Optional<EdgeUrl> parse(@Nullable String url) {
         try {
+            if (null == url) {
+                return Optional.empty();
+            }
+
             return Optional.of(new EdgeUrl(url));
         } catch (URISyntaxException e) {
             return Optional.empty();

code/common/model/src/main/java/nu/marginalia/model/crawl/HtmlFeature.java

@@ -3,11 +3,15 @@ package nu.marginalia.model.crawl;
 import java.util.Collection;
 
 public enum HtmlFeature {
+    // Note, the first 32 of these features are bit encoded in the database
+    // so be sure to keep anything that's potentially important toward the top
+    // of the list
+
     MEDIA( "special:media"),
     JS("special:scripts"),
     AFFILIATE_LINK( "special:affiliate"),
-    TRACKING_INNOCENT("special:tracking"),
-    TRACKING_EVIL("special:tracking2"),
+    TRACKING("special:tracking"),
+    TRACKING_ADTECH("special:ads"), // We'll this as ads for now
 
     VIEWPORT("special:viewport"),
 

code/common/model/src/main/java/nu/marginalia/model/idx/DocumentMetadata.java

@@ -2,6 +2,7 @@ package nu.marginalia.model.idx;
 
 import nu.marginalia.model.crawl.PubDate;
 
+import java.io.Serializable;
 import java.util.EnumSet;
 import java.util.Set;
 
@@ -15,7 +16,9 @@ public record DocumentMetadata(int avgSentLength,
                                int year,
                                int sets,
                                int quality,
-                               byte flags) {
+                               byte flags)
+    implements Serializable
+{
 
     public String toString() {
         StringBuilder sb = new StringBuilder(getClass().getSimpleName());

code/common/process/build.gradle

@@ -20,6 +20,7 @@ dependencies {
 
     implementation libs.guava
     implementation libs.guice
+    implementation libs.bundles.mariadb
     implementation libs.commons.lang3
 
     implementation libs.snakeyaml
@@ -29,4 +30,16 @@ dependencies {
     testImplementation libs.mockito
 }
 
+test {
+    maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+    maxHeapSize = "8G"
+    useJUnitPlatform()
+}
 
+task fastTests(type: Test) {
+    maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+    maxHeapSize = "8G"
+    useJUnitPlatform {
+        excludeTags "slow"
+    }
+}

code/common/process/src/main/java/nu/marginalia/ProcessConfiguration.java

@@ -0,0 +1,7 @@
+package nu.marginalia;
+
+import java.util.UUID;
+
+public record ProcessConfiguration(String processName, int node, UUID instanceUuid) {
+
+}

code/common/process/src/main/java/nu/marginalia/process/control/ProcessHeartbeat.java

@@ -0,0 +1,155 @@
+package nu.marginalia.process.control;
+
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.ProcessConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.concurrent.TimeUnit;
+
+/** This service sends a heartbeat to the database every 5 seconds.
+ */
+@Singleton
+public class ProcessHeartbeat {
+    private final Logger logger = LoggerFactory.getLogger(ProcessHeartbeat.class);
+    private final String processName;
+    private final String processBase;
+    private final String instanceUUID;
+    private final HikariDataSource dataSource;
+
+
+    private final Thread runnerThread;
+    private final int heartbeatInterval = Integer.getInteger("mcp.heartbeat.interval", 1);
+
+    private volatile boolean running = false;
+
+    private volatile int progress = -1;
+
+    @Inject
+    public ProcessHeartbeat(ProcessConfiguration configuration,
+                            HikariDataSource dataSource)
+    {
+        this.processName = configuration.processName() + ":" + configuration.node();
+        this.processBase = configuration.processName();
+        this.dataSource = dataSource;
+
+        this.instanceUUID = configuration.instanceUuid().toString();
+
+        runnerThread = new Thread(this::run);
+
+        Runtime.getRuntime().addShutdownHook(new Thread(this::shutDown));
+    }
+
+    public void setProgress(double progress) {
+        this.progress = (int) (progress * 100);
+    }
+
+    public void start() {
+        if (!running) {
+            runnerThread.start();
+        }
+    }
+
+    public void shutDown() {
+        if (!running)
+            return;
+
+        running = false;
+
+        try {
+            runnerThread.join();
+            heartbeatStop();
+        }
+        catch (InterruptedException|SQLException ex) {
+            logger.warn("ServiceHeartbeat shutdown failed", ex);
+        }
+    }
+
+    private void run() {
+        if (!running)
+            running = true;
+        else
+            return;
+
+        try {
+            heartbeatInit();
+
+            while (running) {
+
+                try {
+                    heartbeatUpdate();
+                }
+                catch (SQLException ex) {
+                    logger.warn("ServiceHeartbeat failed to update", ex);
+                }
+
+                TimeUnit.SECONDS.sleep(heartbeatInterval);
+            }
+        }
+        catch (InterruptedException|SQLException ex) {
+            logger.error("ServiceHeartbeat caught irrecoverable exception, killing service", ex);
+            System.exit(255);
+        }
+    }
+
+    private void heartbeatInit() throws SQLException {
+        try (var connection = dataSource.getConnection()) {
+            try (var stmt = connection.prepareStatement(
+                    """
+                        INSERT INTO PROCESS_HEARTBEAT (PROCESS_NAME, PROCESS_BASE, INSTANCE, HEARTBEAT_TIME, STATUS)
+                        VALUES (?, ?, ?, CURRENT_TIMESTAMP(6), 'STARTING')
+                        ON DUPLICATE KEY UPDATE
+                            INSTANCE = ?,
+                            HEARTBEAT_TIME = CURRENT_TIMESTAMP(6),
+                            STATUS = 'STARTING'
+                        """
+            ))
+            {
+                stmt.setString(1, processName);
+                stmt.setString(2, processBase);
+                stmt.setString(3, instanceUUID);
+                stmt.setString(4, instanceUUID);
+                stmt.executeUpdate();
+            }
+        }
+    }
+
+    private void heartbeatUpdate() throws SQLException {
+        try (var connection = dataSource.getConnection()) {
+            try (var stmt = connection.prepareStatement(
+                    """
+                        UPDATE PROCESS_HEARTBEAT
+                        SET HEARTBEAT_TIME = CURRENT_TIMESTAMP(6), STATUS = 'RUNNING', PROGRESS = ?
+                        WHERE INSTANCE = ?
+                        """)
+            )
+            {
+                stmt.setInt(1, progress);
+                stmt.setString(2, instanceUUID);
+                stmt.executeUpdate();
+            }
+        }
+    }
+
+    private void heartbeatStop() throws SQLException {
+        try (var connection = dataSource.getConnection()) {
+            try (var stmt = connection.prepareStatement(
+                    """
+                        UPDATE PROCESS_HEARTBEAT
+                        SET HEARTBEAT_TIME = CURRENT_TIMESTAMP(6), STATUS='STOPPED', PROGRESS=?
+                        WHERE INSTANCE = ?
+                        """)
+            )
+            {
+                stmt.setInt(1, progress);
+                stmt.setString( 2, instanceUUID);
+                stmt.executeUpdate();
+            }
+        }
+    }
+}
+

code/common/process/src/main/java/nu/marginalia/process/log/WorkLoadIterable.java

@@ -0,0 +1,52 @@
+package nu.marginalia.process.log;
+
+import lombok.SneakyThrows;
+import org.jetbrains.annotations.NotNull;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.Optional;
+import java.util.function.Function;
+
+class WorkLoadIterable<T> implements Iterable<T> {
+
+    private final Path logFile;
+    private final Function<WorkLogEntry, Optional<T>> mapper;
+
+    WorkLoadIterable(Path logFile, Function<WorkLogEntry, Optional<T>> mapper) {
+        this.logFile = logFile;
+        this.mapper = mapper;
+    }
+
+    @NotNull
+    @Override
+    @SneakyThrows
+    public Iterator<T> iterator() {
+        var stream = Files.lines(logFile);
+        return new Iterator<>() {
+            final Iterator<T> iter = stream
+                    .filter(WorkLogEntry::isJobId)
+                    .map(WorkLogEntry::parse)
+                    .map(mapper)
+                    .filter(Optional::isPresent)
+                    .map(Optional::get)
+                    .iterator();
+
+            @Override
+            public boolean hasNext() {
+                if (iter.hasNext()) {
+                    return true;
+                } else {
+                    stream.close();
+                    return false;
+                }
+            }
+
+            @Override
+            public T next() {
+                return iter.next();
+            }
+        };
+    }
+}

code/common/process/src/main/java/nu/marginalia/process/log/WorkLog.java

@@ -1,95 +1,105 @@
 package nu.marginalia.process.log;
 
-import com.google.errorprone.annotations.MustBeClosed;
-import org.apache.logging.log4j.util.Strings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.io.FileNotFoundException;
+import java.io.Closeable;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.LocalDateTime;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.function.Consumer;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;
+import java.util.*;
+import java.util.function.Function;
 
-public class WorkLog implements AutoCloseable {
+/** WorkLog is a journal of work done by a process,
+ * so that it can be resumed after a crash or termination.
+ * <p>
+ * The log file itself is a tab-separated file with the following columns:
+ * <ul>
+ *     <li>Job ID</li>
+ *     <li>Timestamp</li>
+ *     <li>Location (e.g. path on disk)</li>
+ *     <li>Size</li>
+ * </p>
+ *
+ */
+public class WorkLog implements AutoCloseable, Closeable {
     private final Set<String> finishedJobs = new HashSet<>();
     private final FileOutputStream logWriter;
+    private final Logger logger = LoggerFactory.getLogger(getClass());
 
     public WorkLog(Path logFile) throws IOException {
-        loadLog(logFile);
+        if (Files.exists(logFile)) {
+            try (var lines = Files.lines(logFile)) {
+                lines.filter(WorkLogEntry::isJobId)
+                        .map(WorkLogEntry::parseJobIdFromLogLine)
+                        .forEach(finishedJobs::add);
+            }
+        }
 
         logWriter = new FileOutputStream(logFile.toFile(), true);
-        writeLogEntry("# Starting WorkLog @ " + LocalDateTime.now());
+        writeLogEntry("# Starting WorkLog @ " + LocalDateTime.now() + "\n");
     }
 
-    public static void readLog(Path logFile, Consumer<WorkLogEntry> entryConsumer) throws FileNotFoundException {
-        if (!Files.exists(logFile)) {
-            throw new FileNotFoundException("Log file not found " + logFile);
+    /** Create an iterable over the work log
+     * <br>
+     * <b>Caveat: </b> If the iterator is not iterated to the end,
+     *                  it will leak a file descriptor.
+     */
+    public static Iterable<WorkLogEntry> iterable(Path logFile) {
+        return new WorkLoadIterable<>(logFile, Optional::of);
     }
 
-        try (var entries = streamLog(logFile)) {
-            entries.forEach(entryConsumer);
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    @MustBeClosed
-    public static Stream<WorkLogEntry> streamLog(Path logFile) throws IOException {
-        return Files.lines(logFile).filter(WorkLog::isJobId).map(line -> {
-            String[] parts = line.split("\\s+");
-            return new WorkLogEntry(parts[0], parts[1], parts[2], Integer.parseInt(parts[3]));
-        });
-    }
-
-    private void loadLog(Path logFile) throws IOException  {
-        if (!Files.exists(logFile)) {
-            return;
-        }
-
-        try (var lines = Files.lines(logFile)) {
-            lines.filter(WorkLog::isJobId).map(this::getJobIdFromWrittenString).forEach(finishedJobs::add);
-        }
-    }
-
-    private static boolean isJobId(String s) {
-        return Strings.isNotBlank(s) && !s.startsWith("#");
-    }
-
-    private static final Pattern splitPattern = Pattern.compile("\\s+");
-
-    private String getJobIdFromWrittenString(String s) {
-        return splitPattern.split(s, 2)[0];
-    }
-
-    public synchronized boolean isJobFinished(String id) {
-        return finishedJobs.contains(id);
+    /** Create an iterable over the work log, applying a mapping function to each item
+     * <br>
+     * <b>Caveat: </b> If the iterator is not iterated to the end,
+     *                  it will leak a file descriptor.
+     */
+    public static <T> Iterable<T> iterableMap(Path logFile, Function<WorkLogEntry, Optional<T>> mapper) {
+        return new WorkLoadIterable<>(logFile, mapper);
     }
 
     // Use synchro over concurrent set to avoid competing writes
     // - correct is better than fast here, it's sketchy enough to use
     // a PrintWriter
 
+    /** Mark the job as finished in the work log
+     *
+     * @param id  job identifier
+     * @param where  free form field, e.g. location on disk
+     * @param size  free form field, e.g. how many items were processed
+     */
     public synchronized void setJobToFinished(String id, String where, int size) throws IOException {
-        finishedJobs.add(id);
+        if (!finishedJobs.add(id)) {
+            logger.warn("Setting job {} to finished, but it was already finished", id);
+        }
 
-        writeLogEntry(String.format("%s\t%s\t%s\t%d",id, LocalDateTime.now(), where, size));
+        writeLogEntry(String.format("%s\t%s\t%s\t%d\n",id, LocalDateTime.now(), where, size));
+    }
+
+    public synchronized boolean isJobFinished(String id) {
+        return finishedJobs.contains(id);
     }
 
     private void writeLogEntry(String entry) throws IOException {
         logWriter.write(entry.getBytes(StandardCharsets.UTF_8));
-        logWriter.write("\n".getBytes(StandardCharsets.UTF_8));
         logWriter.flush();
     }
 
     @Override
-    public void close() throws Exception {
+    public void close() {
+        try {
             logWriter.flush();
             logWriter.close();
         }
+        catch (IOException e) {
+            logger.error("Error closing work log", e);
+        }
+    }
+
+    public int countFinishedJobs() {
+        return finishedJobs.size();
+    }
 }

code/common/process/src/main/java/nu/marginalia/process/log/WorkLogEntry.java

@@ -1,4 +1,23 @@
 package nu.marginalia.process.log;
 
+import org.apache.logging.log4j.util.Strings;
+
+import java.util.regex.Pattern;
+
 public record WorkLogEntry(String id, String ts, String path, int cnt) {
+    private static final Pattern splitPattern = Pattern.compile("\\s+");
+
+    static WorkLogEntry parse(String line) {
+        String[] parts = splitPattern.split(line);
+        return new WorkLogEntry(parts[0], parts[1], parts[2], Integer.parseInt(parts[3]));
+    }
+
+    static boolean isJobId(String line) {
+        return Strings.isNotBlank(line) && !line.startsWith("#");
+    }
+
+    static String parseJobIdFromLogLine(String s) {
+        return splitPattern.split(s, 2)[0];
+    }
+
 }

code/common/process/src/main/java/nu/marginalia/util/ParallelPipe.java

@@ -1,112 +0,0 @@
-package nu.marginalia.util;
-
-import lombok.SneakyThrows;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/** Generalization of the workflow <br>
- * -- single provider thread reading sequentially from disk <br>
- * -> multiple independent CPU-bound processing tasks <br>
- * -> single consumer thread writing to network/disk <br>
- * <p>
- */
-public abstract class ParallelPipe<INPUT,INTERMEDIATE> {
-    private final LinkedBlockingQueue<INPUT> inputs;
-    private final LinkedBlockingQueue<INTERMEDIATE> intermediates;
-
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    private final List<Thread> processThreads = new ArrayList<>();
-    private final Thread receiverThread;
-
-    private volatile boolean expectingInput = true;
-    private volatile boolean expectingOutput = true;
-
-    public ParallelPipe(String name, int numberOfThreads, int inputQueueSize, int intermediateQueueSize) {
-        inputs = new LinkedBlockingQueue<>(inputQueueSize);
-        intermediates = new LinkedBlockingQueue<>(intermediateQueueSize);
-
-        for (int i = 0; i < numberOfThreads; i++) {
-            processThreads.add(new Thread(this::runProcessThread, name + "-process["+i+"]"));
-        }
-        receiverThread = new Thread(this::runReceiverThread, name + "-receiver");
-
-        processThreads.forEach(Thread::start);
-        receiverThread.start();
-    }
-
-    public void clearQueues() {
-        inputs.clear();
-        intermediates.clear();
-    }
-
-    @SneakyThrows
-    private void runProcessThread() {
-        while (expectingInput || !inputs.isEmpty()) {
-            var in = inputs.poll(10, TimeUnit.SECONDS);
-
-            if (in != null) {
-                try {
-                    var ret = onProcess(in);
-                    if (ret != null) {
-                        intermediates.put(ret);
-                    }
-                }
-                catch (InterruptedException ex) {
-                    throw ex;
-                }
-                catch (Exception ex) {
-                    logger.error("Exception", ex);
-                }
-
-            }
-        }
-
-        logger.info("Terminating {}", Thread.currentThread().getName());
-    }
-
-    @SneakyThrows
-    private void runReceiverThread() {
-        while (expectingOutput || !inputs.isEmpty() || !intermediates.isEmpty()) {
-            var intermediate = intermediates.poll(997, TimeUnit.MILLISECONDS);
-            if (intermediate != null) {
-                try {
-                    onReceive(intermediate);
-                }
-                catch (Exception ex) {
-                    logger.error("Exception", ex);
-                }
-            }
-        }
-
-        logger.info("Terminating {}", Thread.currentThread().getName());
-    }
-
-    /** Begin processing an item */
-    @SneakyThrows
-    public void accept(INPUT input) {
-        inputs.put(input);
-    }
-
-    /** The meat of the processor thread runtime */
-    protected abstract INTERMEDIATE onProcess(INPUT input) throws Exception;
-
-    /** The meat of the consumer thread runtime */
-    protected abstract void onReceive(INTERMEDIATE intermediate) throws Exception;
-
-    public void join() throws InterruptedException {
-        expectingInput = false;
-
-        for (var thread : processThreads) {
-            thread.join();
-        }
-
-        expectingOutput = false;
-        receiverThread.join();
-    }
-}

code/common/process/src/test/java/nu/marginalia/process/log/WorkLogTest.java

@@ -0,0 +1,96 @@
+package nu.marginalia.process.log;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class WorkLogTest {
+
+    private Path logFile;
+    @BeforeEach
+    public void setUp() throws IOException {
+        logFile = Files.createTempFile("worklog", ".log");
+    }
+
+    @AfterEach
+    public void tearDown() throws IOException {
+        Files.deleteIfExists(logFile);
+    }
+
+
+    @Test
+    public void testLog() throws IOException {
+        var log = new WorkLog(logFile);
+        log.setJobToFinished("A", "a.txt",1);
+        log.setJobToFinished("B", "b.txt",2);
+        log.setJobToFinished("C", "c.txt",3);
+        assertTrue(log.isJobFinished("A"));
+        assertTrue(log.isJobFinished("B"));
+        assertTrue(log.isJobFinished("C"));
+        assertFalse(log.isJobFinished("E"));
+    }
+
+    @Test
+    public void testLogResume() throws Exception {
+        WorkLog log = new WorkLog(logFile);
+        log.setJobToFinished("A", "a.txt",1);
+        log.setJobToFinished("B", "b.txt",2);
+        log.setJobToFinished("C", "c.txt",3);
+        log.close();
+        log = new WorkLog(logFile);
+        log.setJobToFinished("E", "e.txt",4);
+        assertTrue(log.isJobFinished("A"));
+        assertTrue(log.isJobFinished("B"));
+        assertTrue(log.isJobFinished("C"));
+        assertTrue(log.isJobFinished("E"));
+        log.close();
+
+        Files.readAllLines(logFile).forEach(System.out::println);
+    }
+
+    @Test
+    public void test() {
+        try (var workLog = new WorkLog(logFile)) {
+            workLog.setJobToFinished("test", "loc1", 4);
+            workLog.setJobToFinished("test2", "loc2", 5);
+            workLog.setJobToFinished("test3", "loc3", 1);
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+
+        try (var workLog = new WorkLog(logFile)) {
+            workLog.setJobToFinished("test4", "loc4", 0);
+
+            assertTrue(workLog.isJobFinished("test"));
+            assertTrue(workLog.isJobFinished("test2"));
+            assertTrue(workLog.isJobFinished("test3"));
+            assertTrue(workLog.isJobFinished("test4"));
+            assertFalse(workLog.isJobFinished("test5"));
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+
+
+        Map<String, WorkLogEntry> entriesById = new HashMap<>();
+        WorkLog.iterable(logFile).forEach(e -> entriesById.put(e.id(), e));
+
+        assertEquals(4, entriesById.size());
+
+        assertEquals("loc1", entriesById.get("test").path());
+        assertEquals("loc2", entriesById.get("test2").path());
+        assertEquals("loc3", entriesById.get("test3").path());
+        assertEquals("loc4", entriesById.get("test4").path());
+
+    }
+}

code/common/service-client/src/main/java/nu/marginalia/client/AbortingScheduler.java

@@ -13,7 +13,6 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
 
 public class AbortingScheduler {
-    private final String name;
     private final ThreadFactory threadFactory;
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
@@ -22,7 +21,6 @@ public class AbortingScheduler {
     private ExecutorService executorService;
 
     public AbortingScheduler(String name) {
-        this.name = name;
         threadFactory = new ThreadFactoryBuilder()
                 .setNameFormat(name+"client--%d")
                 .setUncaughtExceptionHandler(this::handleException)

code/common/service-client/src/main/java/nu/marginalia/client/ServiceMonitors.java

@@ -0,0 +1,133 @@
+package nu.marginalia.client;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.service.id.ServiceId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+@Singleton
+public class ServiceMonitors {
+    private final HikariDataSource dataSource;
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    private final Set<String> runningServices = new HashSet<>();
+    private final Set<Runnable> callbacks = new HashSet<>();
+
+
+    private final int heartbeatInterval = Integer.getInteger("mcp.heartbeat.interval", 5);
+
+    private volatile boolean running;
+
+    @Inject
+    public ServiceMonitors(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+
+        var runThread = new Thread(this::run, "service monitor");
+        runThread.setDaemon(true);
+        runThread.start();
+    }
+
+    public void subscribe(Runnable callback) {
+        synchronized (callbacks) {
+            callbacks.add(callback);
+        }
+    }
+    public void unsubscribe(Runnable callback) {
+        synchronized (callbacks) {
+            callbacks.remove(callback);
+        }
+    }
+
+    public void run() {
+        if (running) {
+            return;
+        }
+        else {
+            running = true;
+        }
+
+        while (running) {
+            if (updateRunningServices()) {
+                runCallbacks();
+            }
+
+            try {
+                TimeUnit.SECONDS.sleep(heartbeatInterval);
+            }
+            catch (InterruptedException ex) {
+                logger.warn("ServiceMonitors interrupted", ex);
+                running = false;
+            }
+        }
+    }
+
+    private void runCallbacks() {
+        synchronized (callbacks) {
+            for (var callback : callbacks) {
+                synchronized (runningServices) {
+                    callback.run();
+                }
+            }
+        }
+    }
+
+    private boolean updateRunningServices() {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                    SELECT SERVICE_BASE, TIMESTAMPDIFF(SECOND, HEARTBEAT_TIME, CURRENT_TIMESTAMP(6))
+                    FROM SERVICE_HEARTBEAT
+                    WHERE ALIVE=1
+                    """)) {
+            try (var rs = stmt.executeQuery()) {
+                Set<String> newRunningServices = new HashSet<>(10);
+                while (rs.next()) {
+                    String svc = rs.getString(1);
+                    int dtime = rs.getInt(2);
+                    if (dtime < 2.5 * heartbeatInterval) {
+                        newRunningServices.add(svc);
+                    }
+                }
+
+                boolean changed;
+
+                synchronized (runningServices) {
+                    changed = !Objects.equals(runningServices, newRunningServices);
+
+                    runningServices.clear();
+                    runningServices.addAll(newRunningServices);
+                }
+
+                return changed;
+            }
+        }
+        catch (SQLException ex) {
+            logger.warn("Failed to update running services", ex);
+        }
+
+        return false;
+    }
+
+    public boolean isServiceUp(ServiceId serviceId) {
+        synchronized (runningServices) {
+            return runningServices.contains(serviceId.name);
+        }
+    }
+
+    public List<ServiceId> getRunningServices() {
+        List<ServiceId> ret = new ArrayList<>(ServiceId.values().length);
+
+        synchronized (runningServices) {
+            for (var runningService : runningServices) {
+                ret.add(ServiceId.byName(runningService));
+            }
+        }
+
+        return ret;
+    }
+}

code/common/service-discovery/src/main/java/nu/marginalia/service/SearchServiceDescriptors.java

@@ -13,5 +13,7 @@ public class SearchServiceDescriptors {
                           new ServiceDescriptor(ServiceId.Search, 5023),
                           new ServiceDescriptor(ServiceId.Assistant, 5025),
                           new ServiceDescriptor(ServiceId.Dating, 5070),
-                          new ServiceDescriptor(ServiceId.Explorer, 5071)));
+                          new ServiceDescriptor(ServiceId.Explorer, 5071),
+                          new ServiceDescriptor(ServiceId.Control, 5090)
+                  ));
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/id/ServiceId.java

@@ -7,19 +7,22 @@ public enum ServiceId {
     Search("search-service"),
     Index("index-service"),
 
+    Control("control-service"),
+
     Dating("dating-service"),
-    Explorer("explorer-service"),
-
-    Other_Auth("auth"),
-    Other_Memex("memex"),
-
-
-    Other_ResourceStore("resource-store"),
-    Other_Renderer("renderer"),
-    Other_PodcastScraper("podcast-scraper");
+    Explorer("explorer-service");
 
     public final String name;
     ServiceId(String name) {
         this.name = name;
     }
+
+    public static ServiceId byName(String name) {
+        for (ServiceId id : values()) {
+            if (id.name.equals(name)) {
+                return id;
+            }
+        }
+        return null;
+    }
 }

code/common/service/build.gradle

@@ -12,6 +12,8 @@ java {
 dependencies {
     implementation project(':code:common:service-client')
     implementation project(':code:common:service-discovery')
+    implementation project(':code:common:message-queue')
+    implementation project(':code:common:db')
 
     implementation libs.lombok
     annotationProcessor libs.lombok

code/common/service/readme.md

@@ -3,6 +3,52 @@
 Contains the base classes for the services. This is where port configuration,
 and common endpoints are set up. 
 
+## Creating a new Service
+
+The minimal service needs a `MainClass` and a `Service` class. 
+
+For proper initiation, the main class should look like this:
+
+```java
+public class FoobarMain extends MainClass {
+
+    @Inject
+    public FoobarMain(FoobarService service) {}
+
+    public static void main(String... args) {
+        init(ServiceId.Foobar, args);
+
+        Injector injector = Guice.createInjector(
+                new FoobarModule(), /* optional custom bindings go here */
+                new DatabaseModule(),
+                new ConfigurationModule(SearchServiceDescriptors.descriptors, 
+                        ServiceId.Foobar));
+
+        injector.getInstance(FoobarMain.class);
+        
+        // set the service as ready so that delayed tasks can be started
+        injector.getInstance(Initialization.class).setReady();
+    }
+}
+```
+
+A service class has a boilerplate set-up that looks like this:
+
+```java
+@Singleton
+public class FoobarService extends Service {
+
+    @Inject
+    public FoobarService(BaseServiceParams params) {
+        super(params);
+        
+        // set up Spark endpoints here
+    }
+}
+```
+
+Further the new service needs to be added to the `ServiceId` enum in [service-discovery](../service-discovery).
+
 ## Central Classes
 
 * [MainClass](src/main/java/nu/marginalia/service/MainClass.java) bootstraps all executables

code/common/service/src/main/java/nu/marginalia/service/MainClass.java

@@ -11,6 +11,9 @@ import org.slf4j.LoggerFactory;
 import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;
 
+/** Each main class of a service should extend this class.
+ *  They must also invoke init() in their main method.
+ */
 public abstract class MainClass {
     private final Logger logger = LoggerFactory.getLogger(getClass());