(zk-registry) epic jak shaving WIP

Cleaning out a lot of old junk from the code, and one thing lead to another...

* Build is improved, now constructing docker images with 'jib'.  Clean build went from 3 minutes to 50 seconds.
* The ProcessService's spawning is smarter.  Will now just spawn a java process instead of relying on the application plugin's generated outputs.
* Project is migrated to GraalVM
* gRPC clients are re-written with a neat fluent/functional style. e.g.
```channelPool.call(grpcStub::method)
              .async(executor) // <-- optional
              .run(argument);
```
This change is primarily to allow handling ManagedChannel errors, but it turned out to be a pretty clean API overall.
* For now the project is all in on zookeeper
* Service discovery is now based on APIs and not services.  Theoretically means we could ship the same code either a monolith or a service mesh.
* To this end, began modularizing a few of the APIs so that they aren't strongly "living" in a service.  WIP!

Missing is documentation and testing, and some more breaking apart of code.

build.gradle

@@ -3,6 +3,10 @@ plugins {
     id("org.jetbrains.gradle.plugin.idea-ext") version "1.0"
     id "io.freefair.lombok" version "8.3"
     id "me.champeau.jmh" version "0.6.6"
+
+    // This is a workaround for a bug in the Jib plugin that causes it to stall randomly
+    // https://github.com/GoogleContainerTools/jib/issues/3347
+    id 'com.google.cloud.tools.jib' version '3.4.0' apply(false)
 }
 
 group 'marginalia'
@@ -29,11 +33,12 @@ subprojects.forEach {it ->
         reproducibleFileOrder = true
     }
 }
-
+ext {
+    dockerImageBase='container-registry.oracle.com/graalvm/jdk:21@sha256:1fd33d4d4eba3a9e1a41a728e39ea217178d257694eea1214fec68d2ed4d3d9b'
+}
 allprojects {
     apply plugin: 'java'
     apply plugin: 'io.freefair.lombok'
-
     dependencies {
         implementation libs.lombok
         testImplementation libs.lombok
@@ -77,3 +82,4 @@ java {
         languageVersion.set(JavaLanguageVersion.of(21))
     }
 }
+

code/api/assistant-api/readme.md

@@ -1,8 +0,0 @@
-# Assistant API
-
-Client and models for talking to the [assistant-service](../../services-core/assistant-service),
-implemented with the base client from  [service-client](../../common/service-client).
-
-## Central Classes
-
-* [AssistantClient](src/main/java/nu/marginalia/assistant/client/AssistantClient.java)

code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java

@@ -1,159 +0,0 @@
-package nu.marginalia.assistant.client;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import nu.marginalia.assistant.api.AssistantApiGrpc;
-import nu.marginalia.assistant.client.model.DictionaryResponse;
-import nu.marginalia.assistant.client.model.DomainInformation;
-import nu.marginalia.assistant.client.model.SimilarDomain;
-import nu.marginalia.service.client.GrpcChannelPoolFactory;
-import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
-import nu.marginalia.service.id.ServiceId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-
-import static nu.marginalia.assistant.client.AssistantProtobufCodec.*;
-
-@Singleton
-public class AssistantClient {
-    private static final Logger logger = LoggerFactory.getLogger(AssistantClient.class);
-
-    private final GrpcSingleNodeChannelPool<AssistantApiGrpc.AssistantApiBlockingStub> channelPool;
-    private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
-    @Inject
-    public AssistantClient(GrpcChannelPoolFactory factory) {
-        this.channelPool = factory.createSingle(ServiceId.Assistant, AssistantApiGrpc::newBlockingStub);
-
-    }
-
-    public Future<DictionaryResponse> dictionaryLookup(String word) {
-        return virtualExecutorService.submit(() -> {
-            var rsp = channelPool.api().dictionaryLookup(
-                    DictionaryLookup.createRequest(word)
-            );
-
-            return DictionaryLookup.convertResponse(rsp);
-        });
-    }
-
-    @SuppressWarnings("unchecked")
-    public Future<List<String>> spellCheck(String word) {
-        return virtualExecutorService.submit(() -> {
-            var rsp = channelPool.api().spellCheck(
-                SpellCheck.createRequest(word)
-            );
-
-            return SpellCheck.convertResponse(rsp);
-        });
-    }
-
-    public Map<String, List<String>> spellCheck(List<String> words, Duration timeout) throws InterruptedException {
-        List<Callable<Map.Entry<String, List<String>>>> tasks = new ArrayList<>();
-
-        for (String w : words) {
-            tasks.add(() -> {
-                var rsp = channelPool.api().spellCheck(
-                        SpellCheck.createRequest(w)
-                );
-
-                return Map.entry(w, SpellCheck.convertResponse(rsp));
-            });
-        }
-
-        var futures = virtualExecutorService.invokeAll(tasks, timeout.toMillis(), TimeUnit.MILLISECONDS);
-        Map<String, List<String>> results = new HashMap<>();
-
-        for (var f : futures) {
-            if (!f.isDone())
-                continue;
-
-            var entry = f.resultNow();
-
-            results.put(entry.getKey(), entry.getValue());
-        }
-
-        return results;
-    }
-
-    public Future<String> unitConversion(String value, String from, String to) {
-        return virtualExecutorService.submit(() -> {
-            var rsp = channelPool.api().unitConversion(
-                    UnitConversion.createRequest(from, to, value)
-            );
-
-            return UnitConversion.convertResponse(rsp);
-        });
-    }
-
-    public Future<String> evalMath(String expression) {
-        return virtualExecutorService.submit(() -> {
-            var rsp = channelPool.api().evalMath(
-                    EvalMath.createRequest(expression)
-            );
-
-            return EvalMath.convertResponse(rsp);
-        });
-    }
-
-    public Future<List<SimilarDomain>> similarDomains(int domainId, int count) {
-        return virtualExecutorService.submit(() -> {
-            try {
-                var rsp = channelPool.api().getSimilarDomains(
-                        DomainQueries.createRequest(domainId, count)
-                );
-
-                return DomainQueries.convertResponse(rsp);
-            }
-            catch (Exception e) {
-                logger.warn("Failed to get similar domains", e);
-
-                throw e;
-            }
-        });
-    }
-
-    public Future<List<SimilarDomain>> linkedDomains(int domainId, int count) {
-        return virtualExecutorService.submit(() -> {
-            try {
-                var rsp = channelPool.api().getLinkingDomains(
-                        DomainQueries.createRequest(domainId, count)
-                );
-
-                return DomainQueries.convertResponse(rsp);
-            }
-            catch (Exception e) {
-                logger.warn("Failed to get linked domains", e);
-                throw e;
-            }
-        });
-
-    }
-
-    public Future<DomainInformation> domainInformation(int domainId) {
-        return virtualExecutorService.submit(() -> {
-            try {
-                var rsp = channelPool.api().getDomainInfo(
-                        DomainInfo.createRequest(domainId)
-                );
-
-                return DomainInfo.convertResponse(rsp);
-            }
-            catch (Exception e) {
-                logger.warn("Failed to get domain information", e);
-
-                throw e;
-            }
-        });
-    }
-
-    public boolean isAccepting() {
-        return channelPool.hasChannel();
-    }
-}

code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java

@@ -13,7 +13,8 @@ import nu.marginalia.executor.upload.UploadDirContents;
 import nu.marginalia.executor.upload.UploadDirItem;
 import nu.marginalia.service.client.GrpcChannelPoolFactory;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
-import nu.marginalia.service.discovery.property.ApiSchema;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.id.ServiceId;
 import nu.marginalia.storage.model.FileStorageId;
 
@@ -22,6 +23,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.net.URISyntaxException;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
@@ -39,180 +41,190 @@ public class ExecutorClient {
     {
         this.registry = registry;
         this.channelPool = grpcChannelPoolFactory
-                .createMulti(ServiceId.Executor, ExecutorApiGrpc::newBlockingStub);
+                .createMulti(
+                        ServiceKey.forGrpcApi(ExecutorApiGrpc.class, ServicePartition.multi()),
+                        ExecutorApiGrpc::newBlockingStub);
     }
 
     public void startFsm(int node, String actorName) {
-        channelPool.apiForNode(node).startFsm(
+        channelPool.call(ExecutorApiBlockingStub::startFsm)
-                RpcFsmName.newBuilder()
+                .forNode(node)
+                .run(RpcFsmName.newBuilder()
                         .setActorName(actorName)
-                        .build()
+                        .build());
-        );
+
     }
 
     public void stopFsm(int node, String actorName) {
-        channelPool.apiForNode(node).stopFsm(
+        channelPool.call(ExecutorApiBlockingStub::stopFsm)
-                RpcFsmName.newBuilder()
+                .forNode(node)
+                .run(RpcFsmName.newBuilder()
                         .setActorName(actorName)
-                        .build()
+                        .build());
-        );
     }
 
     public void stopProcess(int node, String id) {
-        channelPool.apiForNode(node).stopProcess(
+        channelPool.call(ExecutorApiBlockingStub::stopProcess)
-                RpcProcessId.newBuilder()
+                .forNode(node)
+                .run(RpcProcessId.newBuilder()
                         .setProcessId(id)
-                        .build()
+                        .build());
-        );
+
     }
 
     public void triggerCrawl(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).triggerCrawl(
+        channelPool.call(ExecutorApiBlockingStub::triggerCrawl)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
 
     public void triggerRecrawl(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).triggerRecrawl(
+        channelPool.call(ExecutorApiBlockingStub::triggerRecrawl)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
 
     public void triggerConvert(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).triggerConvert(
+        channelPool.call(ExecutorApiBlockingStub::triggerConvert)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
 
     public void triggerConvertAndLoad(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).triggerConvertAndLoad(
+        channelPool.call(ExecutorApiBlockingStub::triggerConvertAndLoad)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
 
     public void loadProcessedData(int node, List<FileStorageId> ids) {
-        channelPool.apiForNode(node).loadProcessedData(
+        channelPool.call(ExecutorApiBlockingStub::loadProcessedData)
-                RpcFileStorageIds.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageIds.newBuilder()
                         .addAllFileStorageIds(ids.stream().map(FileStorageId::id).toList())
-                        .build()
+                        .build());
-        );
     }
 
     public void calculateAdjacencies(int node) {
-        channelPool.apiForNode(node).calculateAdjacencies(Empty.getDefaultInstance());
+        channelPool.call(ExecutorApiBlockingStub::calculateAdjacencies)
+                .forNode(node)
+                .run(Empty.getDefaultInstance());
     }
 
     public void sideloadEncyclopedia(int node, Path sourcePath, String baseUrl) {
-        channelPool.apiForNode(node).sideloadEncyclopedia(
+        channelPool.call(ExecutorApiBlockingStub::sideloadEncyclopedia)
-                RpcSideloadEncyclopedia.newBuilder()
+                .forNode(node)
+                .run(RpcSideloadEncyclopedia.newBuilder()
                         .setBaseUrl(baseUrl)
                         .setSourcePath(sourcePath.toString())
-                        .build()
+                        .build());
-        );
     }
 
     public void sideloadDirtree(int node, Path sourcePath) {
-        channelPool.apiForNode(node).sideloadDirtree(
+        channelPool.call(ExecutorApiBlockingStub::sideloadDirtree)
-                RpcSideloadDirtree.newBuilder()
+                .forNode(node)
+                .run(RpcSideloadDirtree.newBuilder()
                         .setSourcePath(sourcePath.toString())
-                        .build()
+                        .build());
-        );
     }
     public void sideloadReddit(int node, Path sourcePath) {
-        channelPool.apiForNode(node).sideloadReddit(
+        channelPool.call(ExecutorApiBlockingStub::sideloadReddit)
-                RpcSideloadReddit.newBuilder()
+                .forNode(node)
+                .run(RpcSideloadReddit.newBuilder()
                         .setSourcePath(sourcePath.toString())
-                        .build()
+                        .build());
-        );
     }
     public void sideloadWarc(int node, Path sourcePath) {
-        channelPool.apiForNode(node).sideloadWarc(
+        channelPool.call(ExecutorApiBlockingStub::sideloadWarc)
-                RpcSideloadWarc.newBuilder()
+                .forNode(node)
+                .run(RpcSideloadWarc.newBuilder()
                         .setSourcePath(sourcePath.toString())
-                        .build()
+                        .build());
-        );
     }
 
     public void sideloadStackexchange(int node, Path sourcePath) {
-        channelPool.apiForNode(node).sideloadStackexchange(
+        channelPool.call(ExecutorApiBlockingStub::sideloadStackexchange)
-                RpcSideloadStackexchange.newBuilder()
+                .forNode(node)
+                .run(RpcSideloadStackexchange.newBuilder()
                         .setSourcePath(sourcePath.toString())
-                        .build()
+                        .build());
-        );
     }
 
     public void createCrawlSpecFromDownload(int node, String description, String url) {
-        channelPool.apiForNode(node).createCrawlSpecFromDownload(
+        channelPool.call(ExecutorApiBlockingStub::createCrawlSpecFromDownload)
-                RpcCrawlSpecFromDownload.newBuilder()
+                .forNode(node)
+                .run(RpcCrawlSpecFromDownload.newBuilder()
                         .setDescription(description)
                         .setUrl(url)
-                        .build()
+                        .build());
-        );
     }
 
     public void exportAtags(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).exportAtags(
+        channelPool.call(ExecutorApiBlockingStub::exportAtags)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
     public void exportSampleData(int node, FileStorageId fid, int size, String name) {
-        channelPool.apiForNode(node).exportSampleData(
+        channelPool.call(ExecutorApiBlockingStub::exportSampleData)
-                RpcExportSampleData.newBuilder()
+                .forNode(node)
+                .run(RpcExportSampleData.newBuilder()
                         .setFileStorageId(fid.id())
                         .setSize(size)
                         .setName(name)
-                        .build()
+                        .build());
-        );
     }
 
     public void exportRssFeeds(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).exportRssFeeds(
+        channelPool.call(ExecutorApiBlockingStub::exportRssFeeds)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
     public void exportTermFrequencies(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).exportTermFrequencies(
+        channelPool.call(ExecutorApiBlockingStub::exportTermFrequencies)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
 
     public void downloadSampleData(int node, String sampleSet) {
-        channelPool.apiForNode(node).downloadSampleData(
+        channelPool.call(ExecutorApiBlockingStub::downloadSampleData)
-                RpcDownloadSampleData.newBuilder()
+                .forNode(node)
+                .run(RpcDownloadSampleData.newBuilder()
                         .setSampleSet(sampleSet)
-                        .build()
+                        .build());
-        );
     }
 
     public void exportData(int node) {
-        channelPool.apiForNode(node).exportData(Empty.getDefaultInstance());
+        channelPool.call(ExecutorApiBlockingStub::exportData)
+                .forNode(node)
+                .run(Empty.getDefaultInstance());
     }
 
     public void restoreBackup(int node, FileStorageId fid) {
-        channelPool.apiForNode(node).restoreBackup(
+        channelPool.call(ExecutorApiBlockingStub::restoreBackup)
-                RpcFileStorageId.newBuilder()
+                .forNode(node)
+                .run(RpcFileStorageId.newBuilder()
                         .setFileStorageId(fid.id())
-                        .build()
+                        .build());
-        );
     }
 
     public ActorRunStates getActorStates(int node) {
         try {
-            var rs = channelPool.apiForNode(node).getActorStates(Empty.getDefaultInstance());
+            var rs = channelPool.call(ExecutorApiBlockingStub::getActorStates)
+                    .forNode(node)
+                    .run(Empty.getDefaultInstance());
             var states = rs.getActorRunStatesList().stream()
                     .map(r -> new ActorRunState(
                             r.getActorName(),
@@ -236,7 +248,9 @@ public class ExecutorClient {
 
     public UploadDirContents listSideloadDir(int node) {
         try {
-            var rs = channelPool.apiForNode(node).listSideloadDir(Empty.getDefaultInstance());
+            var rs = channelPool.call(ExecutorApiBlockingStub::listSideloadDir)
+                    .forNode(node)
+                    .run(Empty.getDefaultInstance());
             var items = rs.getEntriesList().stream()
                     .map(i -> new UploadDirItem(i.getName(), i.getLastModifiedTime(), i.getIsDirectory(), i.getSize()))
                     .toList();
@@ -252,8 +266,9 @@ public class ExecutorClient {
 
     public FileStorageContent listFileStorage(int node, FileStorageId fileId) {
         try {
-            var rs = channelPool.apiForNode(node).listFileStorage(
+            var rs = channelPool.call(ExecutorApiBlockingStub::listFileStorage)
-                    RpcFileStorageId.newBuilder()
+                    .forNode(node)
+                    .run(RpcFileStorageId.newBuilder()
                             .setFileStorageId(fileId.id())
                             .build()
                     );
@@ -274,13 +289,13 @@ public class ExecutorClient {
         String uriPath = STR."/transfer/file/\{fileId.id()}";
         String uriQuery = STR."path=\{URLEncoder.encode(path, StandardCharsets.UTF_8)}";
 
-        var service = registry.getEndpoints(ApiSchema.REST, ServiceId.Executor, node)
+        var service = registry.getEndpoints(ServiceKey.forRest(ServiceId.Executor, node))
                 .stream().findFirst().orElseThrow();
 
         try (var urlStream = service.endpoint().toURL(uriPath, uriQuery).openStream()) {
             urlStream.transferTo(destOutputStream);
         }
-        catch (IOException ex) {
+        catch (IOException | URISyntaxException ex) {
             throw new RuntimeException(ex);
         }
     }

code/api/index-api/src/main/protobuf/index-api.proto

@@ -4,28 +4,6 @@ package actorapi;
 option java_package="nu.marginalia.index.api";
 option java_multiple_files=true;
 
-service IndexDomainLinksApi {
-  rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {}
-  rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {}
-  rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {}
-  rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {}
-  rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {}
-}
-
-message RpcDomainId {
-  int32 domainId = 1;
-}
-message RpcDomainIdList {
-  repeated int32 domainId = 1 [packed=true];
-}
-message RpcDomainIdCount {
-  int32 idCount = 1;
-}
-message RpcDomainIdPairs {
-  repeated int32 sourceIds = 1 [packed=true];
-  repeated int32 destIds = 2 [packed=true];
-}
-
 service QueryApi {
   rpc query(RpcQsQuery) returns (RpcQsResponse) {}
 }

code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java

@@ -9,14 +9,12 @@ import nu.marginalia.query.model.QueryParams;
 import nu.marginalia.query.model.QueryResponse;
 import nu.marginalia.service.client.GrpcChannelPoolFactory;
 import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
-import nu.marginalia.service.id.ServiceId;
+import nu.marginalia.service.discovery.property.ServiceKey;
-import org.roaringbitmap.longlong.PeekableLongIterator;
+import nu.marginalia.service.discovery.property.ServicePartition;
-import org.roaringbitmap.longlong.Roaring64Bitmap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.annotation.CheckReturnValue;
-import java.util.List;
 
 @Singleton
 public class QueryClient  {
@@ -27,134 +25,25 @@ public class QueryClient  {
             .register();
 
     private final GrpcSingleNodeChannelPool<QueryApiGrpc.QueryApiBlockingStub> queryApiPool;
-    private final GrpcSingleNodeChannelPool<IndexDomainLinksApiGrpc.IndexDomainLinksApiBlockingStub> domainLinkApiPool;
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Inject
     public QueryClient(GrpcChannelPoolFactory channelPoolFactory) {
-        this.queryApiPool = channelPoolFactory.createSingle(ServiceId.Query, QueryApiGrpc::newBlockingStub);
+        this.queryApiPool = channelPoolFactory.createSingle(
-        this.domainLinkApiPool = channelPoolFactory.createSingle(ServiceId.Query, IndexDomainLinksApiGrpc::newBlockingStub);
+                ServiceKey.forGrpcApi(QueryApiGrpc.class, ServicePartition.any()),
+                QueryApiGrpc::newBlockingStub);
     }
 
     @CheckReturnValue
     public QueryResponse search(QueryParams params) {
         var query = QueryProtobufCodec.convertQueryParams(params);
 
-        return wmsa_qs_api_search_time.time(
+        return wmsa_qs_api_search_time.time(() ->
-                () ->  QueryProtobufCodec.convertQueryResponse(queryApiPool
+                QueryProtobufCodec.convertQueryResponse(
-                        .importantCall((api) -> api.query(query))
+                        queryApiPool.call(QueryApiGrpc.QueryApiBlockingStub::query).run(query)
                 )
         );
     }
 
-    public AllLinks getAllDomainLinks() {
-        AllLinks links = new AllLinks();
-
-        domainLinkApiPool.api()
-                .getAllLinks(Empty.getDefaultInstance())
-                .forEachRemaining(pairs -> {
-                    for (int i = 0; i < pairs.getDestIdsCount(); i++) {
-                        links.add(pairs.getSourceIds(i), pairs.getDestIds(i));
-                    }
-                });
-
-        return links;
-    }
-
-    public List<Integer> getLinksToDomain(int domainId) {
-        try {
-            return domainLinkApiPool.api()
-                    .getLinksToDomain(RpcDomainId
-                            .newBuilder()
-                            .setDomainId(domainId)
-                            .build())
-                    .getDomainIdList()
-                    .stream()
-                    .sorted()
-                    .toList();
-        }
-        catch (Exception e) {
-            logger.error("API Exception", e);
-            return List.of();
-        }
-    }
-
-    public List<Integer> getLinksFromDomain(int domainId) {
-        try {
-            return domainLinkApiPool.api()
-                    .getLinksFromDomain(RpcDomainId
-                            .newBuilder()
-                            .setDomainId(domainId)
-                            .build())
-                    .getDomainIdList()
-                    .stream()
-                    .sorted()
-                    .toList();
-        }
-        catch (Exception e) {
-            logger.error("API Exception", e);
-            return List.of();
-        }
-    }
-
-    public int countLinksToDomain(int domainId) {
-        try {
-            return domainLinkApiPool.api()
-                    .countLinksToDomain(RpcDomainId
-                            .newBuilder()
-                            .setDomainId(domainId)
-                            .build())
-                    .getIdCount();
-        }
-        catch (Exception e) {
-            logger.error("API Exception", e);
-            return 0;
-        }
-    }
-
-    public int countLinksFromDomain(int domainId) {
-        try {
-            return domainLinkApiPool.api()
-                    .countLinksFromDomain(RpcDomainId
-                            .newBuilder()
-                            .setDomainId(domainId)
-                            .build())
-                    .getIdCount();
-        }
-        catch (Exception e) {
-            logger.error("API Exception", e);
-            return 0;
-        }
-    }
-    public static class AllLinks {
-        private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap();
-
-        public void add(int source, int dest) {
-            sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest));
-        }
-
-        public Iterator iterator() {
-            return new Iterator();
-        }
-
-        public class Iterator {
-            private final PeekableLongIterator base = sourceToDest.getLongIterator();
-            long val = Long.MIN_VALUE;
-
-            public boolean advance() {
-                if (base.hasNext()) {
-                    val = base.next();
-                    return true;
-                }
-                return false;
-            }
-            public int source() {
-                return (int) (val >>> 32);
-            }
-            public int dest() {
-                return (int) (val & 0xFFFF_FFFFL);
-            }
-        }
-    }
 }

code/common/config/src/main/java/nu/marginalia/WmsaHome.java

@@ -5,8 +5,7 @@ import nu.marginalia.service.ServiceHomeNotConfiguredException;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.util.Objects;
-import java.util.Optional;
 import java.util.stream.Stream;
 
 public class WmsaHome {
@@ -26,15 +25,28 @@ public class WmsaHome {
     }
 
     public static Path getHomePath() {
-        var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElseGet(WmsaHome::findDefaultHomePath);
+        String[] possibleLocations = new String[] {
+            System.getenv("WMSA_HOME"),
+            System.getProperty("system.homePath"),
+            "/var/lib/wmsa",
+            "/wmsa"
+        };
+
+        String retStr = Stream.of(possibleLocations)
+                .filter(Objects::nonNull)
+                .map(Path::of)
+                .filter(Files::isDirectory)
+                .map(Path::toString)
+                .findFirst()
+                .orElseThrow(() ->
+                        new ServiceHomeNotConfiguredException("""
+                            Could not find $WMSA_HOME, either set environment
+                            variable, the 'system.homePath' property,
+                            or ensure either /wmssa or /var/lib/wmsa exists
+                            """));
 
         var ret = Path.of(retStr);
 
-        if (!Files.isDirectory(ret)) {
-            throw new ServiceHomeNotConfiguredException("Could not find $WMSA_HOME, either set environment variable or ensure " + retStr + " exists");
-        }
-
-
         if (!Files.isDirectory(ret.resolve("model"))) {
             throw new ServiceHomeNotConfiguredException("You need to run 'run/setup.sh' to download models to run/ before this will work!");
         }
@@ -42,22 +54,6 @@ public class WmsaHome {
         return ret;
     }
 
-    private static String findDefaultHomePath() {
-
-        // Assume this is a local developer and not a production system, since it would have WMSA_HOME set.
-        // Developers probably have a "run/" somewhere upstream from cwd.
-        //
-
-        return Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent)
-                .filter(p -> Files.exists(p.resolve("run/env")))
-                .filter(p -> Files.exists(p.resolve("run/setup.sh")))
-                .map(p -> p.resolve("run"))
-                .findAny()
-                .orElse(Path.of("/var/lib/wmsa"))
-                .toString();
-    }
-
-
     public static Path getAdsDefinition() {
         return getHomePath().resolve("data").resolve("adblock.txt");
     }

code/common/service-discovery/build.gradle

@@ -6,6 +6,11 @@ plugins {
 repositories {
     mavenLocal()
     mavenCentral()
+
+    repositories {
+        mavenCentral()
+        maven { url 'https://jitpack.io' }
+    }
 }
 
 java {
@@ -18,6 +23,7 @@ dependencies {
 
     implementation libs.bundles.curator
     implementation libs.guice
+    implementation libs.bundles.gson
     implementation libs.bundles.mariadb
     implementation libs.bundles.grpc
     implementation libs.notnull
@@ -29,4 +35,5 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:functions:math:api')
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java

@@ -1,7 +1,6 @@
 package nu.marginalia.service;
 
 import com.google.inject.AbstractModule;
-import nu.marginalia.service.discovery.FixedServiceRegistry;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.discovery.ZkServiceRegistry;
 import org.apache.curator.framework.CuratorFramework;
@@ -18,18 +17,13 @@ public class ServiceDiscoveryModule extends AbstractModule {
     private static final Logger logger = LoggerFactory.getLogger(ServiceDiscoveryModule.class);
 
     public void configure() {
-        getZookeeperHosts().ifPresentOrElse((hosts) -> {
+        var hosts = getZookeeperHosts().orElseThrow(() -> new IllegalStateException("Zookeeper hosts not set"));
         logger.info("Using Zookeeper service registry at {}", hosts);
         CuratorFramework client = CuratorFrameworkFactory
                 .newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000));
 
         bind(CuratorFramework.class).toInstance(client);
         bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class);
-                },
-                () -> {
-                    logger.info("Using fixed service registry");
-                    bind(ServiceRegistryIf.class).to(FixedServiceRegistry.class);
-                });
     }
 
     private Optional<String> getZookeeperHosts() {

code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java

@@ -1,10 +1,6 @@
 package nu.marginalia.service;
 
 public class ServiceHomeNotConfiguredException extends RuntimeException {
-
-    public ServiceHomeNotConfiguredException() {
-        super("WMSA_HOME environment variable not set");
-    }
     public ServiceHomeNotConfiguredException(String message) {
         super(message);
     }

code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java

@@ -6,8 +6,10 @@ import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
 import nu.marginalia.service.NodeConfigurationWatcher;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.discovery.property.PartitionTraits;
 import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
-import nu.marginalia.service.id.ServiceId;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
 
 import java.util.function.Function;
 
@@ -26,30 +28,32 @@ public class GrpcChannelPoolFactory {
     }
 
     /** Create a new multi-node channel pool for the given service. */
-    public <STUB> GrpcMultiNodeChannelPool<STUB> createMulti(ServiceId serviceId,
+    public <STUB> GrpcMultiNodeChannelPool<STUB> createMulti(ServiceKey<ServicePartition.Multi> key,
                                                              Function<ManagedChannel, STUB> stubConstructor)
     {
         return new GrpcMultiNodeChannelPool<>(serviceRegistryIf,
-                serviceId,
+                key,
                 this::createChannel,
                 stubConstructor,
                 nodeConfigurationWatcher);
     }
 
     /** Create a new single-node channel pool for the given service. */
-    public <STUB> GrpcSingleNodeChannelPool<STUB> createSingle(ServiceId serviceId,
+    public <STUB> GrpcSingleNodeChannelPool<STUB> createSingle(ServiceKey<? extends PartitionTraits.Unicast> key,
                                                              Function<ManagedChannel, STUB> stubConstructor)
     {
-        return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, serviceId,
+        return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, key, this::createChannel, stubConstructor);
-                new NodeSelectionStrategy.Any(),
-                this::createChannel,
-                stubConstructor);
     }
 
-    private ManagedChannel createChannel(InstanceAddress<?> route) {
+    private ManagedChannel createChannel(InstanceAddress route) {
-        return ManagedChannelBuilder
+
+        var mc = ManagedChannelBuilder
                 .forAddress(route.host(), route.port())
                 .usePlaintext()
                 .build();
+
+        mc.getState(true);
+
+        return mc;
     }
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java

@@ -4,13 +4,17 @@ import io.grpc.ManagedChannel;
 import lombok.SneakyThrows;
 import nu.marginalia.service.NodeConfigurationWatcher;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.discovery.property.PartitionTraits;
 import nu.marginalia.service.discovery.property.ServiceEndpoint;
-import nu.marginalia.service.id.ServiceId;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.*;
+import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -22,21 +26,20 @@ public class GrpcMultiNodeChannelPool<STUB> {
     private final ConcurrentHashMap<Integer, GrpcSingleNodeChannelPool<STUB>> pools =
             new ConcurrentHashMap<>();
     private static final Logger logger = LoggerFactory.getLogger(GrpcMultiNodeChannelPool.class);
-    private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
     private final ServiceRegistryIf serviceRegistryIf;
-    private final ServiceId serviceId;
+    private final ServiceKey<? extends PartitionTraits.Multicast> serviceKey;
-    private final Function<ServiceEndpoint.InstanceAddress<?>, ManagedChannel> channelConstructor;
+    private final Function<ServiceEndpoint.InstanceAddress, ManagedChannel> channelConstructor;
     private final Function<ManagedChannel, STUB> stubConstructor;
     private final NodeConfigurationWatcher nodeConfigurationWatcher;
 
     @SneakyThrows
     public GrpcMultiNodeChannelPool(ServiceRegistryIf serviceRegistryIf,
-                                    ServiceId serviceId,
+                                    ServiceKey<ServicePartition.Multi> serviceKey,
-                                    Function<ServiceEndpoint.InstanceAddress<?>, ManagedChannel> channelConstructor,
+                                    Function<ServiceEndpoint.InstanceAddress, ManagedChannel> channelConstructor,
                                     Function<ManagedChannel, STUB> stubConstructor,
                                     NodeConfigurationWatcher nodeConfigurationWatcher) {
         this.serviceRegistryIf = serviceRegistryIf;
-        this.serviceId = serviceId;
+        this.serviceKey = serviceKey;
         this.channelConstructor = channelConstructor;
         this.stubConstructor = stubConstructor;
         this.nodeConfigurationWatcher = nodeConfigurationWatcher;
@@ -51,51 +54,74 @@ public class GrpcMultiNodeChannelPool<STUB> {
         return pools.computeIfAbsent(node, _ ->
                 new GrpcSingleNodeChannelPool<>(
                         serviceRegistryIf,
-                        serviceId,
+                        serviceKey.forPartition(ServicePartition.partition(node)),
-                        new NodeSelectionStrategy.Just(node),
                         channelConstructor,
                         stubConstructor));
     }
 
 
-
-    /** Get an API stub for the given node */
-    public STUB apiForNode(int node) {
-        return pools.computeIfAbsent(node, this::getPoolForNode).api();
-    }
-
-
-    /** Invoke a function on each node, returning a list of futures in a terminal state, as per
-     * ExecutorService$invokeAll */
-    public <T> List<Future<T>> invokeAll(Function<STUB, Callable<T>> callF) throws InterruptedException {
-        List<Callable<T>> calls = getEligibleNodes().stream()
-                .mapMulti(this::passNodeIfOk)
-                .map(callF)
-                .toList();
-
-        return virtualExecutorService.invokeAll(calls);
-    }
-
-    /** Invoke a function on each node, returning a stream of results */
-    public <T> Stream<T> callEachSequential(Function<STUB, T> call) {
-        return getEligibleNodes().stream()
-                .mapMulti(this::passNodeIfOk)
-                .map(call);
-    }
-
-    // Eat connectivity exceptions and log them when doing a broadcast-style calls
-    private void passNodeIfOk(Integer nodeId, Consumer<STUB> consumer) {
-        try {
-            consumer.accept(apiForNode(nodeId));
-        }
-        catch (Exception ex) {
-            logger.error("Error calling node {}", nodeId, ex);
-        }
-    }
-
     /** Get the list of nodes that are eligible for broadcast-style requests */
     public List<Integer> getEligibleNodes() {
         return nodeConfigurationWatcher.getQueryNodes();
     }
 
+    public <T, I> CallBuilderBase<T, I> call(BiFunction<STUB, I, T> method) {
+        return new CallBuilderBase<>(method);
+    }
+
+    public class CallBuilderBase<T, I> {
+        private final BiFunction<STUB, I, T> method;
+
+        private CallBuilderBase(BiFunction<STUB, I, T> method) {
+            this.method = method;
+        }
+
+        public GrpcSingleNodeChannelPool<STUB>.CallBuilderBase<T, I> forNode(int node) {
+            return getPoolForNode(node).call(method);
+        }
+
+        public List<T> run(I arg) {
+            return getEligibleNodes().stream()
+                    .map(node -> getPoolForNode(node).call(method).run(arg))
+                    .toList();
+        }
+
+        public CallBuilderAsync<T, I> async(ExecutorService service) {
+            return new CallBuilderAsync<>(service, method);
+        }
+    }
+
+    public class CallBuilderAsync<T, I> {
+        private final Executor executor;
+        private final BiFunction<STUB, I, T> method;
+
+        public CallBuilderAsync(Executor executor, BiFunction<STUB, I, T> method) {
+            this.executor = executor;
+            this.method = method;
+        }
+
+        public CompletableFuture<List<T>> runAll(I arg) {
+            var futures = getEligibleNodes().stream()
+                    .map(GrpcMultiNodeChannelPool.this::getPoolForNode)
+                    .map(pool ->
+                            pool.call(method)
+                                    .async(executor)
+                                    .run(arg)
+                    ).toList();
+
+            return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
+                    .thenApply(v -> futures.stream().map(CompletableFuture::join).toList());
+        }
+
+        public List<CompletableFuture<T>> runEach(I arg) {
+            return getEligibleNodes().stream()
+                    .map(GrpcMultiNodeChannelPool.this::getPoolForNode)
+                    .map(pool ->
+                            pool.call(method)
+                                    .async(executor)
+                                    .run(arg)
+                    ).toList();
+
+        }
+    }
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java

@@ -1,142 +1,232 @@
 package nu.marginalia.service.client;
 
 import com.google.common.collect.Sets;
-import io.grpc.ConnectivityState;
 import io.grpc.ManagedChannel;
 import lombok.SneakyThrows;
 import nu.marginalia.service.discovery.ServiceRegistryIf;
 import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor;
-import nu.marginalia.service.discovery.property.ApiSchema;
+import nu.marginalia.service.discovery.property.PartitionTraits;
 import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
-import nu.marginalia.service.id.ServiceId;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.time.Duration;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 
 /** A pool of gRPC channels for a service, with a separate channel for each node.
  * <p></p>
  * Manages unicast-style requests */
 public class GrpcSingleNodeChannelPool<STUB> extends ServiceChangeMonitor {
-    private final Map<InstanceAddress<?>, ManagedChannel> channels = new ConcurrentHashMap<>();
+    private final Map<InstanceAddress, ConnectionHolder> channels = new ConcurrentHashMap<>();
-    private final Map<Integer, Set<InstanceAddress<?>>> routes = new ConcurrentHashMap<>();
 
     private static final Logger logger = LoggerFactory.getLogger(GrpcSingleNodeChannelPool.class);
 
     private final ServiceRegistryIf serviceRegistryIf;
-    private final ServiceId serviceId;
+    private final Function<InstanceAddress, ManagedChannel> channelConstructor;
-    private final NodeSelectionStrategy nodeSelectionStrategy;
-    private final Function<InstanceAddress<?>, ManagedChannel> channelConstructor;
     private final Function<ManagedChannel, STUB> stubConstructor;
 
+
     @SneakyThrows
     public GrpcSingleNodeChannelPool(ServiceRegistryIf serviceRegistryIf,
-                                     ServiceId serviceId,
+                                     ServiceKey<? extends PartitionTraits.Unicast> serviceKey,
-                                     NodeSelectionStrategy nodeSelectionStrategy,
+                                     Function<InstanceAddress, ManagedChannel> channelConstructor,
-                                     Function<InstanceAddress<?>, ManagedChannel> channelConstructor,
                                      Function<ManagedChannel, STUB> stubConstructor) {
-        super(serviceId);
+        super(serviceKey);
 
         this.serviceRegistryIf = serviceRegistryIf;
-        this.serviceId = serviceId;
-        this.nodeSelectionStrategy = nodeSelectionStrategy;
         this.channelConstructor = channelConstructor;
         this.stubConstructor = stubConstructor;
 
         serviceRegistryIf.registerMonitor(this);
 
         onChange();
+
+        awaitChannel(Duration.ofSeconds(5));
     }
 
 
     @Override
-    public boolean onChange() {
+    public synchronized boolean onChange() {
-        switch (nodeSelectionStrategy) {
+        Set<InstanceAddress> newRoutes = serviceRegistryIf.getEndpoints(serviceKey);
-            case NodeSelectionStrategy.Any() ->
+        Set<InstanceAddress> oldRoutes = new HashSet<>(channels.keySet());
-                    serviceRegistryIf
+
-                        .getServiceNodes(serviceId)
+        // Find the routes that have been added or removed
-                        .forEach(this::refreshNode);
+        for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) {
-            case NodeSelectionStrategy.Just(int node) ->
+            ConnectionHolder oldChannel;
-                    refreshNode(node);
+            if (newRoutes.contains(route)) {
+                logger.info("Adding route {}", route);
+                oldChannel = channels.put(route, new ConnectionHolder(route));
+            } else {
+                logger.info("Expelling route {}", route);
+                oldChannel = channels.remove(route);
+            }
+            if (oldChannel != null) {
+                oldChannel.close();
+            }
         }
 
         return true;
     }
 
-    private void refreshNode(int node) {
+    private class ConnectionHolder implements Comparable<ConnectionHolder> {
+        private final AtomicReference<ManagedChannel> channel = new AtomicReference<>();
+        private final InstanceAddress address;
 
-        Set<InstanceAddress<?>> newRoutes = serviceRegistryIf.getEndpoints(ApiSchema.GRPC, serviceId, node);
+        ConnectionHolder(InstanceAddress address) {
-        Set<InstanceAddress<?>> oldRoutes = routes.getOrDefault(node, Set.of());
+            this.address = address;
-
-        // Find the routes that have been added or removed
-        for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) {
-
-            ManagedChannel oldChannel;
-
-            if (newRoutes.contains(route)) {
-                var newChannel = channelConstructor.apply(route);
-                oldChannel = channels.put(route, newChannel);
-            } else {
-                oldChannel = channels.remove(route);
         }
 
-            if (oldChannel != null)
+        public ManagedChannel get() {
-                oldChannel.shutdown();
+            var value = channel.get();
+            if (value != null) {
+                return value;
             }
 
-        routes.put(node, newRoutes);
+            try {
+                logger.info("Creating channel for {}:{}", serviceKey, address);
+                value = channelConstructor.apply(address);
+                if (channel.compareAndSet(null, value)) {
+                    return value;
                 }
+                else {
+                    value.shutdown();
+                    return channel.get();
+                }
+            }
+            catch (Exception e) {
+                logger.error(STR."Failed to get channel for \{address}", e);
+                return null;
+            }
+        }
+
+        public void close() {
+            ManagedChannel mc = channel.getAndSet(null);
+            if (mc != null) {
+                mc.shutdown();
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            ConnectionHolder that = (ConnectionHolder) o;
+            return Objects.equals(address, that.address);
+        }
+
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(address);
+        }
+
+        @Override
+        public int compareTo(@NotNull GrpcSingleNodeChannelPool<STUB>.ConnectionHolder o) {
+
+            return -Long.compare(address.cxTime(), o.address.cxTime()); // Reverse order
+        }
+    }
+
+
 
     public boolean hasChannel() {
         return !channels.isEmpty();
     }
 
-    /** Get an API stub for the given node */
+    public synchronized boolean awaitChannel(Duration timeout) throws InterruptedException {
-    public STUB api() {
+        if (hasChannel()) return true;
-        return stubConstructor.apply(getChannel());
+
+        final long endTime = System.currentTimeMillis() + timeout.toMillis();
+
+        while (!hasChannel()) {
+            long timeLeft = endTime - System.currentTimeMillis();
+            if (timeLeft <= 0) return false;
+            this.wait(timeLeft);
+        }
+        return hasChannel();
     }
 
-    /** Try to make the call go through.  The function will cycle through
+    private <T, I> T call(BiFunction<STUB, I, T> call, I arg) throws RuntimeException {
-     * available routes until exhaustion, and only then will it give up
+        final List<Exception> exceptions = new ArrayList<>();
-     */
+        final List<ConnectionHolder> connectionHolders = new ArrayList<>(channels.values());
-    public <T> T importantCall(Function<STUB, T> function) {
+
-        for (int i = 0; i < channels.size(); i++) {
+        // Randomize the order of the connection holders to spread out the load
+        Collections.shuffle(connectionHolders);
+
+        for (var channel : connectionHolders) {
             try {
-                return function.apply(api());
+                return call.apply(stubConstructor.apply(channel.get()), arg);
             }
             catch (Exception e) {
-                logger.error("API Exception", e);
+                exceptions.add(e);
             }
         }
 
-        throw new ServiceNotAvailableException(serviceId);
+        for (var e : exceptions) {
+            logger.error("Failed to call service {}", serviceKey, e);
         }
 
-    /** Get the channel that is most ready to use */
+        throw new ServiceNotAvailableException(serviceKey);
-    public ManagedChannel getChannel() {
-        return channels
-                .values()
-                .stream()
-                .min(this::compareChannelsByState)
-                .orElseThrow(() -> new ServiceNotAvailableException(serviceId));
     }
 
-    /** Sort the channels by how ready they are to use */
+    public <T, I> CallBuilderBase<T, I> call(BiFunction<STUB, I, T> method) {
-    private int compareChannelsByState(ManagedChannel a, ManagedChannel b) {
+        return new CallBuilderBase<>(method);
-        var aState = a.getState(true);
-        var bState = b.getState(true);
-
-        if (aState == ConnectivityState.READY) return -1;
-        if (bState == ConnectivityState.READY) return 1;
-        if (aState == ConnectivityState.CONNECTING) return -1;
-        if (bState == ConnectivityState.CONNECTING) return 1;
-        if (aState == ConnectivityState.IDLE) return -1;
-        if (bState == ConnectivityState.IDLE) return 1;
-
-        return 0;
     }
 
+    public class CallBuilderBase<T, I> {
+        private final BiFunction<STUB, I, T> method;
+        private CallBuilderBase(BiFunction<STUB, I, T> method) {
+            this.method = method;
+        }
 
+        public T run(I arg) {
+            return call(method, arg);
+        }
+
+        public List<T> runFor(I... args) {
+            return runFor(List.of(args));
+        }
+
+        public List<T> runFor(List<I> args) {
+            List<T> results = new ArrayList<>();
+            for (var arg : args) {
+                results.add(call(method, arg));
+            }
+            return results;
+        }
+        public CallBuilderAsync<T, I> async(Executor executor) {
+            return new CallBuilderAsync<>(executor, method);
+        }
+    }
+    public class CallBuilderAsync<T, I> {
+        private final Executor executor;
+        private final BiFunction<STUB, I, T> method;
+
+        public CallBuilderAsync(Executor executor, BiFunction<STUB, I, T> method) {
+            this.executor = executor;
+            this.method = method;
+        }
+
+        public CompletableFuture<T> run(I arg) {
+            return CompletableFuture.supplyAsync(() -> call(method, arg), executor);
+        }
+        public CompletableFuture<List<T>> runFor(List<I> args) {
+            List<CompletableFuture<T>> results = new ArrayList<>();
+            for (var arg : args) {
+                results.add(CompletableFuture.supplyAsync(() -> call(method, arg), executor));
+            }
+            return CompletableFuture.allOf(results.toArray(new CompletableFuture[0]))
+                    .thenApply(v -> results.stream().map(CompletableFuture::join).toList());
+        }
+        public CompletableFuture<List<T>> runFor(I... args) {
+            return runFor(List.of(args));
+        }
+    }
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java

@@ -1,17 +0,0 @@
-package nu.marginalia.service.client;
-
-public sealed interface NodeSelectionStrategy {
-    boolean test(int node);
-    record Any() implements NodeSelectionStrategy {
-        @Override
-        public boolean test(int node) {
-            return true;
-        }
-    }
-    record Just(int node) implements NodeSelectionStrategy {
-        @Override
-        public boolean test(int node) {
-            return this.node == node;
-        }
-    }
-}

code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java

@@ -1,12 +1,9 @@
 package nu.marginalia.service.client;
 
-import nu.marginalia.service.id.ServiceId;
+import nu.marginalia.service.discovery.property.ServiceKey;
 
 public class ServiceNotAvailableException extends RuntimeException {
-    public ServiceNotAvailableException(ServiceId id, int node) {
+    public ServiceNotAvailableException(ServiceKey<?> key) {
-        super(STR."Service \{id} not available on node \{node}");
+        super(STR."Service \{key} not available");
-    }
-    public ServiceNotAvailableException(ServiceId id) {
-        super(STR."Service \{id} not available");
     }
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java

@@ -1,122 +0,0 @@
-package nu.marginalia.service.discovery;
-
-import com.google.inject.Inject;
-import com.zaxxer.hikari.HikariDataSource;
-import nu.marginalia.service.discovery.monitor.*;
-import nu.marginalia.service.discovery.property.ApiSchema;
-import nu.marginalia.service.discovery.property.ServiceEndpoint;
-import nu.marginalia.service.discovery.property.ServiceEndpoint.GrpcEndpoint;
-import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress;
-import nu.marginalia.service.discovery.property.ServiceEndpoint.RestEndpoint;
-import nu.marginalia.service.id.ServiceId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.SQLException;
-import java.time.Duration;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.UUID;
-
-/** A service registry that returns fixed endpoints for all services.
- * <p></p>
- * This is for backwards-compatibility with old docker-compose files with no
- * ZooKeeper configured.
- * */
-public class FixedServiceRegistry implements ServiceRegistryIf {
-    private static final Logger logger = LoggerFactory.getLogger(FixedServiceRegistry.class);
-
-    private final HikariDataSource dataSource;
-
-    @Inject
-    public FixedServiceRegistry(HikariDataSource dataSource) {
-        this.dataSource = dataSource;
-    }
-    
-    @Override
-    public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, int node, UUID instanceUUID, String externalAddress) throws Exception {
-        return switch (schema) {
-            case REST -> new ServiceEndpoint.RestEndpoint(externalAddress, 80);
-            case GRPC -> new ServiceEndpoint.GrpcEndpoint(externalAddress, 81);
-        };
-    }
-
-    @Override
-    public void announceInstance(ServiceId id, int node, UUID instanceUUID) {
-        // No-op
-    }
-
-    @Override
-    public Set<Integer> getServiceNodes(ServiceId id) {
-
-        if (id == ServiceId.Executor || id == ServiceId.Index) {
-            try (var conn = dataSource.getConnection();
-                 var stmt = conn.prepareStatement("SELECT ID FROM NODE_CONFIGURATION")) {
-                Set<Integer> ret = new HashSet<>();
-                var rs = stmt.executeQuery();
-                while (rs.next()) {
-                    ret.add(rs.getInt(1));
-                }
-                return ret;
-            }
-            catch (SQLException ex) {
-                return Set.of();
-            }
-        }
-
-        else return Set.of(0);
-    }
-
-    @Override
-    public int requestPort(String externalHost, ApiSchema schema, ServiceId id, int node) {
-        return switch(schema) {
-            case REST -> 80;
-            case GRPC -> 81;
-        };
-    }
-
-    @Override
-    public Set<InstanceAddress<? extends ServiceEndpoint>> getEndpoints(ApiSchema schema, ServiceId id, int node) {
-        return switch (schema) {
-            case REST -> Set.of(new InstanceAddress<>(
-                        new RestEndpoint(id.serviceName + "-" + node, 80),
-                            UUID.randomUUID()));
-            case GRPC -> Set.of(new InstanceAddress<>(
-                        new GrpcEndpoint(id.serviceName + "-" + node, 81),
-                            UUID.randomUUID()));
-        };
-    }
-
-    public void registerMonitor(ServiceMonitorIf monitor) throws Exception {
-        // We don't have any notification mechanism, so we just periodically
-        // invoke the monitor's onChange method to simulate it.
-
-        periodicallyInvoke(monitor, Duration.ofSeconds(15));
-    }
-
-
-    void periodicallyInvoke(ServiceMonitorIf monitor, Duration d) {
-        Thread.ofPlatform().name("PeriodicInvoker").start(() -> {
-            for (;;) {
-                try {
-                    Thread.sleep(d);
-                } catch (InterruptedException e) {
-                    break;
-                }
-
-                boolean reRegister;
-                try {
-                    reRegister = monitor.onChange();
-                }
-                catch (Exception ex) {
-                    logger.error("Monitor failed", ex);
-                    reRegister = true;
-                }
-
-                if (!reRegister) {
-                    break;
-                }
-            }
-        });
-    }
-}

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java

@@ -1,10 +1,10 @@
 package nu.marginalia.service.discovery;
 
 import nu.marginalia.service.discovery.monitor.*;
-import nu.marginalia.service.discovery.property.ApiSchema;
 import nu.marginalia.service.discovery.property.ServiceEndpoint;
 import static nu.marginalia.service.discovery.property.ServiceEndpoint.*;
-import nu.marginalia.service.id.ServiceId;
+
+import nu.marginalia.service.discovery.property.ServiceKey;
 
 import java.util.Set;
 import java.util.UUID;
@@ -16,40 +16,33 @@ public interface ServiceRegistryIf {
     /**
      * Register a service with the registry.
      * <p></p>
-     * Once the instance has announced itself with {@link #announceInstance(ServiceId id, int node, UUID instanceUUID) announceInstance(...)},
+     * Once the instance has announced itself with {@link #announceInstance(UUID instanceUUID) announceInstance(...)},
-     * the service will be available for discovery with {@link #getEndpoints(ApiSchema schema, ServiceId id, int node) getEndpoints(...)}.
+     * the service will be available for discovery with {@link #getEndpoints(ServiceKey key) getEndpoints(...)}.
      *
-     * @param schema          the API schema
+     * @param key             the key identifying the service
-     * @param id              the service identifier
-     * @param node            the node number
      * @param instanceUUID    the unique UUID of the instance
      * @param externalAddress the public address of the service
      */
-    ServiceEndpoint registerService(ApiSchema schema,
+    ServiceEndpoint registerService(ServiceKey<?> key,
-                                    ServiceId id,
-                                    int node,
                                     UUID instanceUUID,
                                     String externalAddress) throws Exception;
 
+
+    void declareFirstBoot();
+    void waitForFirstBoot() throws InterruptedException;
+
     /** Let the world know that the service is running
      * and ready to accept requests. */
-    void announceInstance(ServiceId id, int node, UUID instanceUUID);
+    void announceInstance(UUID instanceUUID);
-
-    /** Return all nodes that are running for the specified service. */
-    Set<Integer> getServiceNodes(ServiceId id);
 
     /** At the discretion of the implementation, provide a port that is unique
-     * across (externalHost, serviceId, schema, node).  It may be randomly selected
+     * across (host, api-schema).  It may be randomly selected
      * or hard-coded or some combination of behaviors.
      */
-    int requestPort(String externalHost,
+    int requestPort(String externalHost, ServiceKey<?> key);
-                    ApiSchema schema,
-                    ServiceId id,
-                    int node);
 
     /** Get all endpoints for the service on the specified node and schema. */
-    Set<InstanceAddress<? extends ServiceEndpoint>>
+    Set<InstanceAddress> getEndpoints(ServiceKey<?> schema);
-        getEndpoints(ApiSchema schema, ServiceId id, int node);
 
     /** Register a monitor to be notified when the service registry changes.
      * <p></p>
@@ -61,9 +54,6 @@ public interface ServiceRegistryIf {
      * monitor type.
      * <ul>
      * <li>{@link ServiceChangeMonitor} is notified when any node for the service changes.</li>
-     * <li>{@link ServiceNodeChangeMonitor} is notified when a specific node for the service changes.</li>
-     * <li>{@link ServiceRestEndpointChangeMonitor} is notified when the REST endpoints for the specified node service changes.</li>
-     * <li>{@link ServiceGrpcEndpointChangeMonitor} is notified when the gRPC endpoints for the specified node service changes.</li>
      * </ul>
      * */
     void registerMonitor(ServiceMonitorIf monitor) throws Exception;

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java

@@ -4,10 +4,10 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import lombok.SneakyThrows;
 import nu.marginalia.service.discovery.monitor.*;
-import nu.marginalia.service.discovery.property.ApiSchema;
 import nu.marginalia.service.discovery.property.ServiceEndpoint;
 import static nu.marginalia.service.discovery.property.ServiceEndpoint.*;
-import nu.marginalia.service.id.ServiceId;
+
+import nu.marginalia.service.discovery.property.ServiceKey;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.api.CuratorWatcher;
 import org.apache.curator.utils.ZKPaths;
@@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /** A versatile service registry that uses ZooKeeper to store service endpoints.
  * It is used to register services and to look up the endpoints of other services.
@@ -35,6 +34,8 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
     private static final Logger logger = LoggerFactory.getLogger(ZkServiceRegistry.class);
     private volatile boolean stopped = false;
 
+    private final List<String> livenessPaths = new ArrayList<>();
+
     @Inject
     @SneakyThrows
     public ZkServiceRegistry(CuratorFramework curatorFramework) {
@@ -51,93 +52,98 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
     }
 
     @Override
-    public ServiceEndpoint registerService(ApiSchema schema, ServiceId id,
+    public ServiceEndpoint registerService(ServiceKey<?> key,
-                                           int node,
                                            UUID instanceUUID,
                                            String externalAddress)
     throws Exception
     {
-        var ephemeralProperty = curatorFramework.create()
+        var endpoint = new ServiceEndpoint(externalAddress, requestPort(externalAddress, key));
-                .creatingParentsIfNeeded()
-                .withMode(CreateMode.EPHEMERAL);
 
-        var endpoint = ServiceEndpoint.forSchema(schema, externalAddress,
+        String path = STR."\{key.toPath()}/\{instanceUUID.toString()}";
-                requestPort(externalAddress, schema, id, node)
+        byte[] payload = STR."\{endpoint.host()}:\{endpoint.port()}".getBytes(StandardCharsets.UTF_8);
-        );
-
-        String path;
-        byte[] payload;
-
-        switch (endpoint) {
-            case ServiceEndpoint.GrpcEndpoint(String host, int port) -> {
-                path = STR."/services/\{id.serviceName}/\{node}/grpc/\{instanceUUID.toString()}";
-                payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8);
-            }
-            case ServiceEndpoint.RestEndpoint(String host, int port) -> {
-                path = STR."/services/\{id.serviceName}/\{node}/rest/\{instanceUUID.toString()}";
-                payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8);
-            }
-        }
 
         logger.info("Registering {} -> {}", path, endpoint);
 
-        ephemeralProperty.forPath(path, payload);
+        curatorFramework.create()
+                .creatingParentsIfNeeded()
+                .withMode(CreateMode.EPHEMERAL)
+                .forPath(path, payload);
 
         return endpoint;
     }
 
+    @SneakyThrows
     @Override
-    public void announceInstance(ServiceId id, int node, UUID instanceUUID) {
+    public void declareFirstBoot() {
+        if (!isFirstBoot()) {
+            curatorFramework.create()
+                    .creatingParentsIfNeeded()
+                    .withMode(CreateMode.PERSISTENT)
+                    .forPath(STR."/first-boot");
+        }
+    }
+
+    @Override
+    public void waitForFirstBoot() throws InterruptedException {
+        if (!isFirstBoot())
+            logger.info("Waiting for first-boot flag");
+
+        while (true) {
+            if (isFirstBoot())
+                return;
+
+            Thread.sleep(1000);
+        }
+    }
+
+    private boolean isFirstBoot() {
         try {
-            String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}";
+            return curatorFramework.checkExists().forPath("/first-boot") != null;
+        }
+        catch (Exception ex) {
+            logger.error("Failed to check first-boot", ex);
+            return false;
+        }
+    }
+
+    @Override
+    public void announceInstance(UUID instanceUUID) {
+        try {
+            String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}";
+
+            livenessPaths.add(serviceRoot);
+
             curatorFramework.create()
                     .creatingParentsIfNeeded()
                     .withMode(CreateMode.EPHEMERAL)
                     .forPath(serviceRoot);
         }
         catch (Exception ex) {
-            logger.error("Failed to create service root for {}", id.serviceName);
+            logger.error("Failed to create service root for {}", instanceUUID);
         }
     }
 
     /**
      * Returns true if the service has announced itself as up and running.
      */
-    public boolean isInstanceRunning(ServiceId id, int node, UUID instanceUUID) {
+    public boolean isInstanceRunning(UUID instanceUUID) {
         try {
-            String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}";
+            String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}";
             return null != curatorFramework.checkExists().forPath(serviceRoot);
         }
         catch (Exception ex) {
-            logger.error("Failed to check if service is running {}", id.serviceName);
+            logger.error("Failed to check if instance is running {}", instanceUUID);
             return false;
         }
     }
 
-    @Override
-    public Set<Integer> getServiceNodes(ServiceId id) {
-        try {
-            String serviceRoot = STR."/services/\{id.serviceName}";
-            return curatorFramework.getChildren().forPath(serviceRoot)
-                    .stream().map(Integer::parseInt)
-                    .collect(Collectors.toSet());
-        }
-        catch (Exception ex) {
-            logger.error("Failed to get nodes for service {}", id.serviceName);
-            return Set.of();
-        }
-    }
-
     @Override
     public int requestPort(String externalHost,
-                           ApiSchema schema,
+                           ServiceKey<?> key) {
-                           ServiceId id,
-                           int node)
-    {
         if (!Boolean.getBoolean("service.random-port")) {
-            return switch(schema) {
+            return switch (key) {
-                case REST -> 80;
+                case ServiceKey.Rest rest -> 80;
-                case GRPC -> 81;
+                case ServiceKey.Grpc<?> grpc -> 81;
             };
         }
 
@@ -146,9 +152,9 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
 
         var random = new Random();
 
-        String host = STR."\{id.serviceName}-\{node}";
+        String identifier = key.toPath();
 
-        byte[] payload = STR."\{schema}://\{host}".getBytes(StandardCharsets.UTF_8);
+        byte[] payload = identifier.getBytes();
 
         for (int iter = 0; iter < 1000; iter++) {
             try {
@@ -161,7 +167,7 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
                 return port;
             }
             catch (Exception ex) {
-                logger.error(STR."Still negotiating port for \{schema}://\{id.serviceName}:\{node}");
+                logger.error(STR."Still negotiating port for \{identifier}");
             }
         }
 
@@ -169,81 +175,30 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
     }
 
     @Override
-    public Set<InstanceAddress<? extends ServiceEndpoint>> getEndpoints(ApiSchema schema, ServiceId id, int node) {
+    public Set<InstanceAddress> getEndpoints(ServiceKey<?> key) {
-        return switch (schema) {
-            case REST -> getRestEndpoints(id, node);
-            case GRPC -> getGrpcEndpoints(id, node);
-        };
-    }
-
-    public Set<InstanceAddress<? extends ServiceEndpoint>> getRestEndpoints(ServiceId id, int node) {
         try {
-            Set<InstanceAddress<? extends ServiceEndpoint>> ret = new HashSet<>();
+            Set<InstanceAddress> ret = new HashSet<>();
-            String restRoot = STR."/services/\{id.serviceName}/\{node}/rest";
             for (var uuid : curatorFramework
                     .getChildren()
-                    .forPath(restRoot)) {
+                    .forPath(key.toPath())) {
 
-                if (!isInstanceRunning(id, node, UUID.fromString(uuid))) {
+                if (!isInstanceRunning(UUID.fromString(uuid))) {
                     continue;
                 }
 
+                var path = ZKPaths.makePath(key.toPath(), uuid);
                 byte[] data = curatorFramework
                         .getData()
-                        .forPath(ZKPaths.makePath(restRoot, uuid));
+                        .forPath(path);
-                String hostAndPort = new String(data);
-                var address = RestEndpoint
-                        .parse(hostAndPort)
-                        .asInstance(UUID.fromString(uuid));
 
-                // Ensure that the address is resolvable
+                long cxTime = curatorFramework.checkExists().forPath(path).getMzxid();
-                // (this reduces the risk of exceptions when trying to connect to the service)
-                if (!address.endpoint().validateHost()) {
-                    logger.warn("Omitting stale address {}, address does not resolve", address);
-                    continue;
-                }
-
-                ret.add(address);
-
-            }
-
-            return ret;
-        }
-        catch (Exception ex) {
-            return Set.of();
-        }
-    }
-
-    public Set<InstanceAddress<? extends ServiceEndpoint>> getGrpcEndpoints(ServiceId id, int node) {
-        try {
-            Set<InstanceAddress<? extends ServiceEndpoint>> ret = new HashSet<>();
-            String restRoot = STR."/services/\{id.serviceName}/\{node}/grpc";
-            for (var uuid : curatorFramework
-                    .getChildren()
-                    .forPath(restRoot)) {
-
-                if (!isInstanceRunning(id, node, UUID.fromString(uuid))) {
-                    continue;
-                }
-
-                byte[] data = curatorFramework
-                        .getData()
-                        .forPath(ZKPaths.makePath(restRoot, uuid));
 
                 String hostAndPort = new String(data);
-                var address = GrpcEndpoint
+                var address = ServiceEndpoint
                         .parse(hostAndPort)
-                        .asInstance(UUID.fromString(uuid));
+                        .asInstance(UUID.fromString(uuid), cxTime);
-
-                // Ensure that the address is resolvable
-                // (this reduces the risk of exceptions when trying to connect to the service)
-                if (!address.endpoint().validateHost()) {
-                    logger.warn("Omitting stale address {}, address does not resolve", address);
-                    continue;
-                }
 
                 ret.add(address);
-
             }
 
             return ret;
@@ -254,29 +209,13 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
     }
 
     public void registerMonitor(ServiceMonitorIf monitor) throws Exception {
-        monitor.register(this);
+        if (stopped)
-    }
+            logger.info("Not registering monitor for {} because the registry is stopped", monitor.getKey());
 
-    public void registerMonitor(ServiceChangeMonitor monitor) throws Exception {
+        String path = monitor.getKey().toPath();
-        installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}");
-    }
 
-    public void registerMonitor(ServiceNodeChangeMonitor monitor) throws Exception {
+        CuratorWatcher watcher = change -> {
-        installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}");
-    }
-
-    public void registerMonitor(ServiceRestEndpointChangeMonitor monitor) throws Exception {
-        installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/rest");
-    }
-
-    public void registerMonitor(ServiceGrpcEndpointChangeMonitor monitor) throws Exception {
-        installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/grpc");
-    }
-
-    private void installMonitor(ServiceMonitorIf monitor, String path) throws Exception {
-        CuratorWatcher watcher = _ -> {
             boolean reRegister;
-
             try {
                 reRegister = monitor.onChange();
             }
@@ -293,13 +232,31 @@ public class ZkServiceRegistry implements ServiceRegistryIf {
         curatorFramework.watchers().add()
                 .usingWatcher(watcher)
                 .forPath(path);
+
+        // Also register for updates to the running-instances list,
+        // as this will have an effect on the result of getEndpoints()
+        curatorFramework.watchers().add()
+                .usingWatcher(watcher)
+                .forPath("/running-instances");
     }
 
     /* Exposed for tests */
     public synchronized void shutDown() {
-        if (!stopped) {
+        if (!stopped)
-            curatorFramework.close();
+            return;
+
         stopped = true;
+
+        // Delete all liveness paths
+        for (var path : livenessPaths) {
+            logger.info("Cleaning up {}", path);
+
+            try {
+                curatorFramework.delete().forPath(path);
+            }
+            catch (Exception ex) {
+                logger.error("Failed to delete path {}", path, ex);
+            }
         }
     }
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java

@@ -1,23 +1,17 @@
 package nu.marginalia.service.discovery.monitor;
 
-import nu.marginalia.service.discovery.ServiceRegistryIf;
+import nu.marginalia.service.discovery.property.ServiceKey;
-import nu.marginalia.service.discovery.ZkServiceRegistry;
-import nu.marginalia.service.id.ServiceId;
 
 public abstract class ServiceChangeMonitor implements ServiceMonitorIf {
-    public final ServiceId serviceId;
+    public final ServiceKey<?> serviceKey;
 
-    public ServiceChangeMonitor(ServiceId serviceId) {
+    public ServiceChangeMonitor(ServiceKey<?> key) {
-        this.serviceId = serviceId;
+        this.serviceKey = key;
     }
 
     public abstract boolean onChange();
-    public void register(ServiceRegistryIf registry) throws Exception {
+    public ServiceKey<?> getKey() {
-        if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
+        return serviceKey;
-            zkServiceRegistry.registerMonitor(this);
-        }
-        else {
-            registry.registerMonitor(this);
-        }
     }
+
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java

@@ -1,25 +0,0 @@
-package nu.marginalia.service.discovery.monitor;
-
-import nu.marginalia.service.discovery.ServiceRegistryIf;
-import nu.marginalia.service.discovery.ZkServiceRegistry;
-import nu.marginalia.service.id.ServiceId;
-
-public abstract class ServiceGrpcEndpointChangeMonitor implements ServiceMonitorIf {
-    public final ServiceId serviceId;
-    public final int node;
-    public ServiceGrpcEndpointChangeMonitor(ServiceId serviceId, int node) {
-        this.serviceId = serviceId;
-        this.node = node;
-    }
-
-    public abstract boolean onChange();
-
-    public void register(ServiceRegistryIf registry) throws Exception {
-        if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
-            zkServiceRegistry.registerMonitor(this);
-        }
-        else {
-            registry.registerMonitor(this);
-        }
-    }
-}

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java

@@ -1,16 +1,13 @@
 package nu.marginalia.service.discovery.monitor;
 
-import nu.marginalia.service.discovery.ServiceRegistryIf;
+
+import nu.marginalia.service.discovery.property.ServiceKey;
 
 public interface ServiceMonitorIf {
     /** Called when the monitored service has changed.
      * @return true if the monitor is to be refreshed
      */
     boolean onChange();
+    ServiceKey<?> getKey();
 
-    /** Register this monitor with the given registry.
-     * It is preferred to use {@link ServiceRegistryIf}'s
-     * registerMonitor function.
-     * */
-    void register(ServiceRegistryIf registry) throws Exception;
 }

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java

@@ -1,25 +0,0 @@
-package nu.marginalia.service.discovery.monitor;
-
-import nu.marginalia.service.discovery.ServiceRegistryIf;
-import nu.marginalia.service.discovery.ZkServiceRegistry;
-import nu.marginalia.service.id.ServiceId;
-
-public abstract class ServiceNodeChangeMonitor implements ServiceMonitorIf {
-    public final ServiceId serviceId;
-    public final int node;
-    public ServiceNodeChangeMonitor(ServiceId serviceId, int node) {
-        this.serviceId = serviceId;
-        this.node = node;
-    }
-
-    public abstract boolean onChange();
-
-    public void register(ServiceRegistryIf registry) throws Exception {
-        if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
-            zkServiceRegistry.registerMonitor(this);
-        }
-        else {
-            registry.registerMonitor(this);
-        }
-    }
-}

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java

@@ -1,25 +0,0 @@
-package nu.marginalia.service.discovery.monitor;
-
-import nu.marginalia.service.discovery.ServiceRegistryIf;
-import nu.marginalia.service.discovery.ZkServiceRegistry;
-import nu.marginalia.service.id.ServiceId;
-
-public abstract class ServiceRestEndpointChangeMonitor implements ServiceMonitorIf {
-    public final ServiceId serviceId;
-    public final int node;
-    public ServiceRestEndpointChangeMonitor(ServiceId serviceId, int node) {
-        this.serviceId = serviceId;
-        this.node = node;
-    }
-
-    public abstract boolean onChange();
-
-    public void register(ServiceRegistryIf registry) throws Exception {
-        if (registry instanceof ZkServiceRegistry zkServiceRegistry) {
-            zkServiceRegistry.registerMonitor(this);
-        }
-        else {
-            registry.registerMonitor(this);
-        }
-    }
-}

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java

@@ -1,6 +0,0 @@
-package nu.marginalia.service.discovery.property;
-
-public enum ApiSchema {
-    REST,
-    GRPC
-}

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java

@@ -0,0 +1,8 @@
+package nu.marginalia.service.discovery.property;
+
+public interface PartitionTraits {
+    interface Grpc {};
+    interface Unicast {};
+    interface Multicast {};
+    interface NoGrpc {};
+}

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java

@@ -1,17 +1,23 @@
 package nu.marginalia.service.discovery.property;
 
-
-import lombok.SneakyThrows;
-
 import java.net.*;
 import java.util.UUID;
 
-public sealed interface ServiceEndpoint {
+public record ServiceEndpoint(String host, int port) {
-    String host();
-    int port();
 
-    URL toURL(String endpoint, String query);
+    public static ServiceEndpoint parse(String hostAndPort) {
-    default InetSocketAddress toInetSocketAddress() {
+        var parts = hostAndPort.split(":");
+        if (parts.length != 2) {
+            throw new IllegalArgumentException("Invalid host:port string: " + hostAndPort);
+        }
+        return new ServiceEndpoint(parts[0], Integer.parseInt(parts[1]));
+    }
+
+    public URL toURL(String endpoint, String query) throws URISyntaxException, MalformedURLException {
+        return new URI("http", null, host, port, endpoint, query, null)
+                .toURL();
+    }
+    public InetSocketAddress toInetSocketAddress() {
          return new InetSocketAddress(host(), port());
     }
 
@@ -19,7 +25,7 @@ public sealed interface ServiceEndpoint {
      *
      * @return true if the host is a valid
      */
-    default boolean validateHost() {
+    public boolean validateHost() {
         try {
             // Throws UnknownHostException if the host is not a valid IP address or hostname
             // (this should not be slow since the DNS lookup should be local, and if it isn't;
@@ -31,63 +37,11 @@ public sealed interface ServiceEndpoint {
         }
     }
 
-    static ServiceEndpoint forSchema(ApiSchema schema, String host, int port) {
+    public InstanceAddress asInstance(UUID instance, long cxTime) {
-        return switch (schema) {
+        return new InstanceAddress(this, instance, cxTime);
-            case REST -> new RestEndpoint(host, port);
-            case GRPC -> new GrpcEndpoint(host, port);
-        };
     }
 
-    record RestEndpoint(String host, int port) implements ServiceEndpoint {
+    public record InstanceAddress(ServiceEndpoint endpoint, UUID instance, long cxTime) {
-        public static RestEndpoint parse(String hostColonPort) {
-            String[] parts = hostColonPort.split(":");
-
-            if (parts.length != 2) {
-                throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'");
-            }
-
-            return new RestEndpoint(
-                    parts[0],
-                    Integer.parseInt(parts[1])
-            );
-        }
-
-        @SneakyThrows
-        public URL toURL(String endpoint, String query) {
-            return new URI("http", null, host, port, endpoint, query, null)
-                    .toURL();
-        }
-
-        public InstanceAddress<RestEndpoint> asInstance(UUID uuid) {
-            return new InstanceAddress<>(this, uuid);
-        }
-    }
-
-    record GrpcEndpoint(String host, int port) implements ServiceEndpoint {
-        public static GrpcEndpoint parse(String hostColonPort) {
-            String[] parts = hostColonPort.split(":");
-
-            if (parts.length != 2) {
-                throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'");
-            }
-
-            return new GrpcEndpoint(
-                    parts[0],
-                    Integer.parseInt(parts[1])
-            );
-        }
-
-        public InstanceAddress<GrpcEndpoint> asInstance(UUID uuid) {
-            return new InstanceAddress<>(this, uuid);
-        }
-
-        @Override
-        public URL toURL(String endpoint, String query) {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-    record InstanceAddress<T extends ServiceEndpoint>(T endpoint, UUID instance) {
         public String host() {
             return endpoint.host();
         }

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java

@@ -0,0 +1,69 @@
+package nu.marginalia.service.discovery.property;
+
+import io.grpc.ServiceDescriptor;
+import nu.marginalia.service.id.ServiceId;
+
+public sealed interface ServiceKey<P extends ServicePartition> {
+    String toPath();
+
+    static ServiceKey<ServicePartition.None> forRest(ServiceId id) {
+        return new Rest(id.serviceName);
+    }
+    static ServiceKey<ServicePartition.None> forRest(ServiceId id, int node) {
+        if (node == 0) {
+            return forRest(id);
+        }
+
+        return new Rest(id.serviceName + "-" + node);
+    }
+
+    static Grpc<ServicePartition> forServiceDescriptor(ServiceDescriptor descriptor, ServicePartition partition) {
+        return new Grpc<>(descriptor.getName(), partition);
+    }
+
+    static <P2 extends ServicePartition & PartitionTraits.Grpc> Grpc<P2> forGrpcApi(Class<?> apiClass, P2 partition) {
+        try {
+            var name = apiClass.getField("SERVICE_NAME").get(null);
+            return new Grpc<P2>(name.toString(), partition);
+        }
+        catch (Exception e) {
+            throw new IllegalArgumentException("Could not get SERVICE_NAME from " + apiClass.getSimpleName(), e);
+        }
+    }
+
+
+    <P2 extends ServicePartition & PartitionTraits.Grpc & PartitionTraits.Unicast>
+    Grpc<P2> forPartition(P2 partition);
+
+
+    record Rest(String name) implements ServiceKey<ServicePartition.None> {
+        public String toPath() {
+            return STR."/services/rest/\{name}";
+        }
+
+        @Override
+        public
+        <P2 extends ServicePartition & PartitionTraits.Grpc & PartitionTraits.Unicast>
+            Grpc<P2> forPartition(P2 partition)
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+    record Grpc<P extends ServicePartition>(String name, P partition) implements ServiceKey<P> {
+        public String baseName() {
+            return STR."/services/grpc/\{name}";
+        }
+        public String toPath() {
+            return STR."/services/grpc/\{name}/\{partition.identifier()}";
+        }
+
+        @Override
+        public
+        <P2 extends ServicePartition & PartitionTraits.Grpc & PartitionTraits.Unicast>
+            Grpc<P2> forPartition(P2 partition)
+        {
+            return new Grpc<>(name, partition);
+        }
+    }
+
+}

code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java

@@ -0,0 +1,29 @@
+package nu.marginalia.service.discovery.property;
+
+public sealed interface ServicePartition {
+    String identifier();
+
+    static Any any() { return new Any(); }
+    static Multi multi() { return new Multi(); }
+    static Partition partition(int node) { return new Partition(node); }
+    static None none() { return new None(); }
+
+    record Any() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast {
+        public String identifier() { return "*"; }
+
+    }
+    record Multi() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Multicast {
+        public String identifier() { return "*"; }
+
+    }
+    record Partition(int node) implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast {
+        public String identifier() {
+            return Integer.toString(node);
+        }
+
+    }
+    record None() implements ServicePartition, PartitionTraits.NoGrpc {
+        public String identifier() { return ""; }
+
+    }
+}

code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java

@@ -1,6 +1,8 @@
 package nu.marginalia.service.discovery;
 
-import nu.marginalia.service.discovery.property.ApiSchema;
+import nu.marginalia.api.math.MathApiGrpc;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.id.ServiceId;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
@@ -13,7 +15,6 @@ import org.testcontainers.junit.jupiter.Testcontainers;
 
 import java.util.*;
 
-import static nu.marginalia.service.discovery.property.ServiceEndpoint.*;
 import static org.junit.jupiter.api.Assertions.*;
 
 @Testcontainers
@@ -58,15 +59,18 @@ class ZkServiceRegistryTest {
 
         List<Integer> ports = new ArrayList<>();
         Set<Integer> portsSet = new HashSet<>();
+
+        var key = ServiceKey.forRest(ServiceId.Search, 0);
+
         for (int i = 0; i < 500; i++) {
-            int port = registry1.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0);
+            int port = registry1.requestPort("127.0.0.1", key);
             ports.add(port);
 
             // Ensure we get unique ports
             assertTrue(portsSet.add(port));
         }
         for (int i = 0; i < 50; i++) {
-            int port = registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0);
+            int port = registry2.requestPort("127.0.0.1", key);
             ports.add(port);
 
             // Ensure we get unique ports
@@ -75,39 +79,86 @@ class ZkServiceRegistryTest {
         registry1.shutDown();
         for (int i = 0; i < 500; i++) {
             // Verify we can reclaim ports
-            ports.add(registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0));
+            ports.add(registry2.requestPort("127.0.0.1", key));
         }
         assertEquals(1050, ports.size());
     }
 
     @Test
-    void getInstances() throws Exception {
+    void getInstancesRestgRPC() throws Exception {
         var uuid1 = UUID.randomUUID();
         var uuid2 = UUID.randomUUID();
 
         var registry1 = createRegistry();
         var registry2 = createRegistry();
 
-        var endpoint1 = (RestEndpoint) registry1.registerService(ApiSchema.REST, ServiceId.Search, 0, uuid1, "127.0.0.1");
+        var key1 = ServiceKey.forRest(ServiceId.Search, 0);
-        var endpoint2 = (GrpcEndpoint) registry2.registerService(ApiSchema.GRPC, ServiceId.Search, 0, uuid2, "127.0.0.2");
+        var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any());
 
-        registry1.announceInstance(ServiceId.Search, 0, uuid1);
+        var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1");
-        registry2.announceInstance(ServiceId.Search, 0, uuid2);
+        var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2");
 
-        assertEquals(Set.of(endpoint1.asInstance(uuid1)),
+        registry1.announceInstance(uuid1);
-                registry1.getRestEndpoints(ServiceId.Search, 0));
+        registry2.announceInstance(uuid2);
 
-        assertEquals(Set.of(endpoint2.asInstance(uuid2)),
+        assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)),
-                registry1.getGrpcEndpoints(ServiceId.Search, 0));
+                registry1.getEndpoints(key1));
 
+        assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)),
+                registry1.getEndpoints(key2));
 
         registry1.shutDown();
         Thread.sleep(100);
 
-        assertEquals(Set.of(),
+        assertEquals(Set.of(), registry2.getEndpoints(key1));
-                registry2.getRestEndpoints(ServiceId.Search, 0));
+        assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key2));
-        assertEquals(Set.of(endpoint2.asInstance(uuid2)),
+    }
-                registry2.getGrpcEndpoints(ServiceId.Search, 0));
+
+    @Test
+    void testInstancesTwoAny() throws Exception {
+        var uuid1 = UUID.randomUUID();
+        var uuid2 = UUID.randomUUID();
+
+        var registry1 = createRegistry();
+        var registry2 = createRegistry();
+
+        var key = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any());
+
+        var endpoint1 = registry1.registerService(key, uuid1, "127.0.0.1");
+        var endpoint2 = registry2.registerService(key, uuid2, "127.0.0.2");
+
+        registry1.announceInstance(uuid1);
+        registry2.announceInstance(uuid2);
+
+        assertEquals(Set.of(endpoint1.asInstance(uuid1, 0),
+                        endpoint2.asInstance(uuid2, 0)),
+                registry1.getEndpoints(key));
+
+        registry1.shutDown();
+        Thread.sleep(100);
+
+        assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key));
+    }
+
+    @Test
+    void testInstancesTwoPartitions() throws Exception {
+        var uuid1 = UUID.randomUUID();
+        var uuid2 = UUID.randomUUID();
+
+        var registry1 = createRegistry();
+        var registry2 = createRegistry();
+
+        var key1 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(1));
+        var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(2));
+
+        var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1");
+        var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2");
+
+        registry1.announceInstance(uuid1);
+        registry2.announceInstance(uuid2);
+
+        assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), registry1.getEndpoints(key1));
+        assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry1.getEndpoints(key2));
     }
 
     @Test
@@ -115,9 +166,9 @@ class ZkServiceRegistryTest {
         var registry1 = createRegistry();
         var uuid1 = UUID.randomUUID();
 
-        assertFalse(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1));
+        assertFalse(registry1.isInstanceRunning(uuid1));
-        registry1.announceInstance(ServiceId.Search, 0, uuid1);
+        registry1.announceInstance(uuid1);
-        assertTrue(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1));
+        assertTrue(registry1.isInstanceRunning(uuid1));
 
         registry1.shutDown();
     }

code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java

@@ -156,6 +156,8 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat {
                 stmt.executeUpdate();
             }
         }
+
+        dataSource.close();
     }
 
 }

code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java

@@ -6,6 +6,7 @@ import com.google.inject.Singleton;
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
+import nu.marginalia.WmsaHome;
 import nu.marginalia.service.ServiceHomeNotConfiguredException;
 import org.flywaydb.core.Flyway;
 import org.mariadb.jdbc.Driver;
@@ -51,7 +52,7 @@ public class DatabaseModule extends AbstractModule {
     }
 
     private Properties loadDbProperties() {
-        Path propDir = getHomePath().resolve("conf/db.properties");
+        Path propDir = WmsaHome.getHomePath().resolve("conf/db.properties");
         if (!Files.isRegularFile(propDir)) {
             throw new IllegalStateException("Database properties file " + propDir + " does not exist");
         }
@@ -72,17 +73,6 @@ public class DatabaseModule extends AbstractModule {
 
     }
 
-    public static Path getHomePath() {
-        var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse("/var/lib/wmsa");
-
-        var ret = Path.of(retStr);
-        if (!Files.isDirectory(ret)) {
-            throw new ServiceHomeNotConfiguredException("Could not find WMSA_HOME, either set environment variable or ensure /var/lib/wmsa exists");
-        }
-        return ret;
-    }
-
-
     @SneakyThrows
     @Singleton
     @Provides
@@ -97,7 +87,6 @@ public class DatabaseModule extends AbstractModule {
         try {
             HikariConfig config = new HikariConfig();
 
-
             config.setJdbcUrl(connStr);
             config.setUsername(dbProperties.getProperty(DB_USER_KEY));
             config.setPassword(dbProperties.getProperty(DB_PASS_KEY));

code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java

@@ -1,5 +1,6 @@
 package nu.marginalia.service.module;
 
+import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.id.ServiceId;
 
 import java.util.UUID;

code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java

@@ -65,7 +65,7 @@ public class ServiceConfigurationModule extends AbstractModule {
         }
 
         // If we're in docker, we'll use the hostname
-        if (isDocker()) {
+        if (Boolean.getBoolean("service.useDockerHostname")) {
             return System.getenv("HOSTNAME");
         }
 
@@ -82,14 +82,7 @@ public class ServiceConfigurationModule extends AbstractModule {
             return configuredValue;
         }
 
-        // If we're in docker, we'll bind to all interfaces
-        if (isDocker())
-            return "0.0.0.0";
-        else  // If we're not in docker, we'll default to binding to localhost to avoid exposing services
         return "127.0.0.1";
     }
 
-    boolean isDocker() {
-        return System.getenv("WMSA_IN_DOCKER") != null;
-    }
 }

code/common/service/src/main/java/nu/marginalia/service/server/Service.java

@@ -5,8 +5,8 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
 import io.prometheus.client.Counter;
 import lombok.SneakyThrows;
 import nu.marginalia.mq.inbox.*;
-import nu.marginalia.service.discovery.property.ApiSchema;
+import nu.marginalia.service.discovery.property.*;
-import nu.marginalia.service.discovery.property.ServiceEndpoint;
+import nu.marginalia.service.id.ServiceId;
 import nu.marginalia.service.server.mq.ServiceMqSubscription;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -16,6 +16,7 @@ import spark.Request;
 import spark.Response;
 import spark.Spark;
 
+import java.net.InetSocketAddress;
 import java.util.List;
 import java.util.Optional;
 
@@ -48,11 +49,24 @@ public class Service {
     @SneakyThrows
     public Service(BaseServiceParams params,
                    Runnable configureStaticFiles,
+                   ServicePartition partition,
                    List<BindableService> grpcServices) {
+
         this.initialization = params.initialization;
         var config = params.configuration;
         node = config.node();
 
+        if (config.serviceId() == ServiceId.Control) {
+            // Special case for first boot, since the control service
+            // owns database migrations and so on, we need other processes
+            // to wait for this to be done before they start.  This is
+            // only needed once.
+            params.serviceRegistry.declareFirstBoot();
+        }
+        else {
+            params.serviceRegistry.waitForFirstBoot();
+        }
+
         String inboxName = config.serviceName();
         logger.info("Inbox name: {}", inboxName);
 
@@ -60,8 +74,7 @@ public class Service {
 
         var restEndpoint =
                 serviceRegistry.registerService(
-                        ApiSchema.REST, config.serviceId(),
+                        ServiceKey.forRest(config.serviceId(), config.node()),
-                    config.node(),
                     config.instanceUuid(),
                     config.externalAddress()
                 );
@@ -75,7 +88,7 @@ public class Service {
         initialization.addCallback(params.heartbeat::start);
         initialization.addCallback(messageQueueInbox::start);
         initialization.addCallback(() -> params.eventLog.logEvent("SVC-INIT", serviceName + ":" + config.node()));
-        initialization.addCallback(() -> serviceRegistry.announceInstance(config.serviceId(), config.node(), config.instanceUuid()));
+        initialization.addCallback(() -> serviceRegistry.announceInstance(config.instanceUuid()));
 
         if (!initialization.isReady() && ! initialized ) {
             initialized = true;
@@ -101,29 +114,39 @@ public class Service {
             Spark.get("/internal/started", this::isInitialized);
             Spark.get("/internal/ready", this::isReady);
 
-            ServiceEndpoint.GrpcEndpoint grpcEndpoint = (ServiceEndpoint.GrpcEndpoint) params.serviceRegistry.registerService(
+            int port = params.serviceRegistry.requestPort(config.externalAddress(), new ServiceKey.Grpc<>("-", partition));
-                    ApiSchema.GRPC, config.serviceId(),
+
-                    config.node(),
+            // Start the gRPC server
+            var grpcServerBuilder = NettyServerBuilder.forAddress(new InetSocketAddress(config.bindAddress(), port));
+            for (var grpcService : grpcServices) {
+                var svc = grpcService.bindService();
+
+                params.serviceRegistry.registerService(
+                        ServiceKey.forServiceDescriptor(svc.getServiceDescriptor(), partition),
                         config.instanceUuid(),
                         config.externalAddress()
                         );
 
-            // Start the gRPC server
+                grpcServerBuilder.addService(svc);
-            var grpcServerBuilder = NettyServerBuilder.forAddress(grpcEndpoint.toInetSocketAddress());
-            for (var grpcService : grpcServices) {
-                grpcServerBuilder.addService(grpcService);
             }
             grpcServerBuilder.build().start();
         }
     }
 
     public Service(BaseServiceParams params,
+                   ServicePartition partition,
                    List<BindableService> grpcServices) {
-        this(params, Service::defaultSparkConfig, grpcServices);
+        this(params,
+                Service::defaultSparkConfig,
+                partition,
+                grpcServices);
     }
 
     public Service(BaseServiceParams params) {
-        this(params, Service::defaultSparkConfig, List.of());
+        this(params,
+                Service::defaultSparkConfig,
+                ServicePartition.any(),
+                List.of());
     }
 
     private static void defaultSparkConfig() {

code/common/service/src/main/resources/log4j2-prod.xml

@@ -21,6 +21,8 @@
         </RollingFile>
     </Appenders>
     <Loggers>
+        <Logger name="org.apache.zookeeper" level="WARN" />
+
         <Root level="info">
             <AppenderRef ref="Console"/>
             <AppenderRef ref="LogToFile"/>

code/common/service/src/main/resources/log4j2-test.xml

@@ -20,6 +20,8 @@
         </RollingFile>
     </Appenders>
     <Loggers>
+        <Logger name="org.apache.zookeeper" level="WARN" />
+
         <Root level="info">
             <AppenderRef ref="Console"/>
             <AppenderRef ref="LogToFile"/>

code/features-index/domain-ranking/build.gradle

@@ -17,7 +17,7 @@ dependencies {
     implementation project(':code:common:db')
     implementation project(':code:common:model')
     implementation project(':code:common:service')
-    implementation project(':code:api:query-api')
+    implementation project(':code:functions:domain-links:api')
 
     implementation 'org.jgrapht:jgrapht-core:1.5.2'
 

code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java

@@ -3,23 +3,20 @@ package nu.marginalia.ranking.data;
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
-import nu.marginalia.query.client.QueryClient;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import org.jgrapht.Graph;
 import org.jgrapht.graph.DefaultDirectedGraph;
 import org.jgrapht.graph.DefaultEdge;
 
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
 
 /** A source for the inverted link graph,
  * which is the same as the regular graph except
  * the direction of the links have been inverted */
 public class InvertedLinkGraphSource extends AbstractGraphSource {
-    private final QueryClient queryClient;
+    private final AggregateDomainLinksClient queryClient;
 
     @Inject
-    public InvertedLinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) {
+    public InvertedLinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient queryClient) {
         super(dataSource);
         this.queryClient = queryClient;
     }

code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java

@@ -3,19 +3,19 @@ package nu.marginalia.ranking.data;
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
-import nu.marginalia.query.client.QueryClient;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import org.jgrapht.Graph;
 import org.jgrapht.graph.DefaultDirectedGraph;
 import org.jgrapht.graph.DefaultEdge;
 
 /** A source for the regular link graph. */
 public class LinkGraphSource extends AbstractGraphSource {
-    private final QueryClient queryClient;
+    private final AggregateDomainLinksClient domainLinksClient;
 
     @Inject
-    public LinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) {
+    public LinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) {
         super(dataSource);
-        this.queryClient = queryClient;
+        this.domainLinksClient = domainLinksClient;
     }
 
     @SneakyThrows
@@ -25,7 +25,7 @@ public class LinkGraphSource extends AbstractGraphSource {
 
         addVertices(graph);
 
-        var allLinks = queryClient.getAllDomainLinks();
+        var allLinks = domainLinksClient.getAllDomainLinks();
         var iter = allLinks.iterator();
         while (iter.advance()) {
             if (!graph.containsVertex(iter.dest())) {

code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java

@@ -3,7 +3,7 @@ package nu.marginalia.ranking;
 
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
-import nu.marginalia.query.client.QueryClient;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import nu.marginalia.ranking.data.InvertedLinkGraphSource;
 import nu.marginalia.ranking.data.LinkGraphSource;
 import nu.marginalia.ranking.data.SimilarityGraphSource;
@@ -37,8 +37,9 @@ public class RankingAlgorithmsContainerTest {
 
     static HikariDataSource dataSource;
 
-    QueryClient queryClient;
+    AggregateDomainLinksClient domainLinksClient;
-    QueryClient.AllLinks allLinks;
+    AggregateDomainLinksClient.AllLinks allLinks;
+
     @BeforeAll
     public static void setup() {
         HikariConfig config = new HikariConfig();
@@ -66,9 +67,9 @@ public class RankingAlgorithmsContainerTest {
 
     @BeforeEach
     public void setupQueryClient() {
-        queryClient = Mockito.mock(QueryClient.class);
+        domainLinksClient = Mockito.mock(AggregateDomainLinksClient.class);
-        allLinks = new QueryClient.AllLinks();
+        allLinks = new AggregateDomainLinksClient.AllLinks();
-        when(queryClient.getAllDomainLinks()).thenReturn(allLinks);
+        when(domainLinksClient.getAllDomainLinks()).thenReturn(allLinks);
 
         try (var conn = dataSource.getConnection();
              var stmt = conn.createStatement()) {
@@ -97,7 +98,7 @@ public class RankingAlgorithmsContainerTest {
     @Test
     public void testGetDomains() {
         // should all be the same, doesn't matter which one we use
-        var source = new LinkGraphSource(dataSource, queryClient);
+        var source = new LinkGraphSource(dataSource, domainLinksClient);
 
         Assertions.assertEquals(List.of(1),
                 source.domainIds(List.of("memex.marginalia.nu")));
@@ -111,7 +112,7 @@ public class RankingAlgorithmsContainerTest {
     public void testLinkGraphSource() {
         allLinks.add(1, 3);
 
-        var graph = new LinkGraphSource(dataSource, queryClient).getGraph();
+        var graph = new LinkGraphSource(dataSource, domainLinksClient).getGraph();
 
         Assertions.assertTrue(graph.containsVertex(1));
         Assertions.assertTrue(graph.containsVertex(2));
@@ -127,7 +128,7 @@ public class RankingAlgorithmsContainerTest {
     public void testInvertedLinkGraphSource() {
         allLinks.add(1, 3);
 
-        var graph = new InvertedLinkGraphSource(dataSource, queryClient).getGraph();
+        var graph = new InvertedLinkGraphSource(dataSource, domainLinksClient).getGraph();
 
         Assertions.assertTrue(graph.containsVertex(1));
         Assertions.assertTrue(graph.containsVertex(2));

code/functions/domain-info/api/build.gradle

@@ -0,0 +1,45 @@
+plugins {
+    id 'java'
+
+    id "com.google.protobuf" version "0.9.4"
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
+}
+
+jar.archiveBaseName = 'domain-info-api'
+
+sourceSets {
+    main {
+        proto {
+            srcDir 'src/main/protobuf'
+        }
+    }
+}
+
+apply from: "$rootProject.projectDir/protobuf.gradle"
+
+dependencies {
+    implementation project(':code:common:model')
+    implementation project(':code:common:config')
+    implementation project(':code:common:service-discovery')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.notnull
+    implementation libs.guice
+    implementation libs.gson
+    implementation libs.protobuf
+    implementation libs.javax.annotation
+    implementation libs.bundles.grpc
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+}

code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java

@@ -0,0 +1,56 @@
+package nu.marginalia.api.domains;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.api.domains.model.SimilarDomain;
+import nu.marginalia.service.client.GrpcChannelPoolFactory;
+import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.*;
+
+import nu.marginalia.api.domains.model.*;
+
+@Singleton
+public class DomainInfoClient {
+    private static final Logger logger = LoggerFactory.getLogger(DomainInfoClient.class);
+
+    private final GrpcSingleNodeChannelPool<DomainInfoAPIGrpc.DomainInfoAPIBlockingStub> channelPool;
+    private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
+
+    @Inject
+    public DomainInfoClient(GrpcChannelPoolFactory factory) {
+        this.channelPool = factory.createSingle(
+                ServiceKey.forGrpcApi(DomainInfoAPIGrpc.class, ServicePartition.any()),
+                DomainInfoAPIGrpc::newBlockingStub);
+    }
+
+    public Future<List<SimilarDomain>> similarDomains(int domainId, int count) {
+        return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getSimilarDomains)
+                .async(virtualExecutorService)
+                .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count))
+                .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse);
+    }
+
+    public Future<List<SimilarDomain>> linkedDomains(int domainId, int count) {
+        return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getLinkingDomains)
+                .async(virtualExecutorService)
+                .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count))
+                .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse);
+    }
+
+    public Future<DomainInformation> domainInformation(int domainId) {
+        return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getDomainInfo)
+                .async(virtualExecutorService)
+                .run(DomainsProtobufCodec.DomainInfo.createRequest(domainId))
+                .thenApply(DomainsProtobufCodec.DomainInfo::convertResponse);
+    }
+
+    public boolean isAccepting() {
+        return channelPool.hasChannel();
+    }
+}

code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java

@@ -1,74 +1,14 @@
-package nu.marginalia.assistant.client;
+package nu.marginalia.api.domains;
 
 import lombok.SneakyThrows;
-import nu.marginalia.assistant.api.*;
-import nu.marginalia.assistant.client.model.DictionaryEntry;
-import nu.marginalia.assistant.client.model.DictionaryResponse;
-import nu.marginalia.assistant.client.model.DomainInformation;
-import nu.marginalia.assistant.client.model.SimilarDomain;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.EdgeUrl;
+import nu.marginalia.api.domains.model.*;
 
 import java.util.ArrayList;
 import java.util.List;
 
-public class AssistantProtobufCodec {
+public class DomainsProtobufCodec {
-
-    public static class DictionaryLookup {
-        public static RpcDictionaryLookupRequest createRequest(String word) {
-            return RpcDictionaryLookupRequest.newBuilder()
-                    .setWord(word)
-                    .build();
-        }
-        public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) {
-            return new DictionaryResponse(
-                    rsp.getWord(),
-                    rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList()
-            );
-        }
-
-        private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) {
-            return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition());
-        }
-    }
-
-    public static class SpellCheck {
-        public static RpcSpellCheckRequest createRequest(String text) {
-            return RpcSpellCheckRequest.newBuilder()
-                    .setText(text)
-                    .build();
-        }
-
-        public static List<String> convertResponse(RpcSpellCheckResponse rsp) {
-            return rsp.getSuggestionsList();
-        }
-    }
-
-    public static class UnitConversion {
-        public static RpcUnitConversionRequest createRequest(String from, String to, String unit) {
-            return RpcUnitConversionRequest.newBuilder()
-                    .setFrom(from)
-                    .setTo(to)
-                    .setUnit(unit)
-                    .build();
-        }
-
-        public static String convertResponse(RpcUnitConversionResponse rsp) {
-            return rsp.getResult();
-        }
-    }
-
-    public static class EvalMath {
-        public static RpcEvalMathRequest createRequest(String expression) {
-            return RpcEvalMathRequest.newBuilder()
-                    .setExpression(expression)
-                    .build();
-        }
-
-        public static String convertResponse(RpcEvalMathResponse rsp) {
-            return rsp.getResult();
-        }
-    }
 
     public static class DomainQueries {
         public static RpcDomainLinksRequest createRequest(int domainId, int count) {

code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.client.model;
+package nu.marginalia.api.domains.model;
 
 import lombok.*;
 import nu.marginalia.model.EdgeDomain;

code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.client.model;
+package nu.marginalia.api.domains.model;
 
 import nu.marginalia.model.EdgeUrl;
 

code/functions/domain-info/api/src/main/protobuf/domain-info.proto

@@ -1,18 +1,10 @@
 syntax="proto3";
-package assistantapi;
+package marginalia.api.domain;
 
-option java_package="nu.marginalia.assistant.api";
+option java_package="nu.marginalia.api.domains";
 option java_multiple_files=true;
 
-service AssistantApi {
+service DomainInfoAPI {
-  /** Looks up a word in the dictionary. */
-  rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {}
-  /** Checks the spelling of a text. */
-  rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {}
-  /** Converts a unit from one to another. */
-  rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {}
-  /** Evaluates a mathematical expression. */
-  rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {}
 
   /** Fetches information about a domain. */
   rpc getDomainInfo(RpcDomainId) returns (RpcDomainInfoResponse) {}

code/functions/domain-info/build.gradle

@@ -0,0 +1,44 @@
+plugins {
+    id 'java'
+
+    id 'application'
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
+}
+
+dependencies {
+    implementation project(':code:functions:domain-info:api')
+    implementation project(':code:functions:domain-links:api')
+
+    implementation project(':code:common:config')
+    implementation project(':code:common:service')
+    implementation project(':code:common:model')
+    implementation project(':code:common:db')
+    implementation project(':code:common:service-discovery')
+
+    implementation project(':code:libraries:geo-ip')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.bundles.grpc
+    implementation libs.notnull
+    implementation libs.guice
+    implementation libs.spark
+    implementation libs.opencsv
+    implementation libs.trove
+    implementation libs.fastutil
+    implementation libs.bundles.gson
+    implementation libs.bundles.mariadb
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+
+}

code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java

@@ -0,0 +1,53 @@
+package nu.marginalia.functions.domains;
+
+import com.google.inject.Inject;
+import io.grpc.stub.StreamObserver;
+import nu.marginalia.api.domains.DomainInfoAPIGrpc;
+import nu.marginalia.api.domains.*;
+
+public class DomainInfoGrpcService extends DomainInfoAPIGrpc.DomainInfoAPIImplBase {
+
+    private final DomainInformationService domainInformationService;
+    private final SimilarDomainsService similarDomainsService;
+    @Inject
+    public DomainInfoGrpcService(DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService)
+    {
+
+        this.domainInformationService = domainInformationService;
+        this.similarDomainsService = similarDomainsService;
+    }
+
+    @Override
+    public void getDomainInfo(RpcDomainId request, StreamObserver<RpcDomainInfoResponse> responseObserver) {
+        var ret = domainInformationService.domainInfo(request.getDomainId());
+
+        ret.ifPresent(responseObserver::onNext);
+
+        responseObserver.onCompleted();
+    }
+
+    @Override
+    public void getSimilarDomains(RpcDomainLinksRequest request,
+                                  StreamObserver<RpcSimilarDomains> responseObserver) {
+        var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount());
+
+        var responseBuilder = RpcSimilarDomains
+                .newBuilder()
+                .addAllDomains(ret);
+
+        responseObserver.onNext(responseBuilder.build());
+        responseObserver.onCompleted();
+    }
+
+    @Override
+    public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver<RpcSimilarDomains> responseObserver) {
+        var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount());
+
+        var responseBuilder = RpcSimilarDomains
+                .newBuilder()
+                .addAllDomains(ret);
+
+        responseObserver.onNext(responseBuilder.build());
+        responseObserver.onCompleted();
+    }
+}

code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java

@@ -1,11 +1,11 @@
-package nu.marginalia.assistant.domains;
+package nu.marginalia.functions.domains;
 
 import com.zaxxer.hikari.HikariDataSource;
-import nu.marginalia.assistant.api.RpcDomainInfoResponse;
+import nu.marginalia.api.domains.RpcDomainInfoResponse;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import nu.marginalia.geoip.GeoIpDictionary;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.db.DbDomainQueries;
-import nu.marginalia.query.client.QueryClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,7 +21,7 @@ public class DomainInformationService {
     private final GeoIpDictionary geoIpDictionary;
 
     private DbDomainQueries dbDomainQueries;
-    private final QueryClient queryClient;
+    private final AggregateDomainLinksClient domainLinksClient;
     private HikariDataSource dataSource;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
@@ -29,11 +29,11 @@ public class DomainInformationService {
     public DomainInformationService(
             DbDomainQueries dbDomainQueries,
             GeoIpDictionary geoIpDictionary,
-            QueryClient queryClient,
+            AggregateDomainLinksClient domainLinksClient,
             HikariDataSource dataSource) {
         this.dbDomainQueries = dbDomainQueries;
         this.geoIpDictionary = geoIpDictionary;
-        this.queryClient = queryClient;
+        this.domainLinksClient = domainLinksClient;
         this.dataSource = dataSource;
     }
 
@@ -84,8 +84,8 @@ public class DomainInformationService {
             inCrawlQueue = rs.next();
             builder.setInCrawlQueue(inCrawlQueue);
 
-            builder.setIncomingLinks(queryClient.countLinksToDomain(domainId));
+            builder.setIncomingLinks(domainLinksClient.countLinksToDomain(domainId));
-            builder.setOutboundLinks(queryClient.countLinksFromDomain(domainId));
+            builder.setOutboundLinks(domainLinksClient.countLinksFromDomain(domainId));
 
             rs = stmt.executeQuery(STR."""
                     SELECT KNOWN_URLS, GOOD_URLS, VISITED_URLS FROM DOMAIN_METADATA WHERE ID=\{domainId}

code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.domains;
+package nu.marginalia.functions.domains;
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
@@ -8,10 +8,10 @@ import gnu.trove.map.hash.TIntDoubleHashMap;
 import gnu.trove.map.hash.TIntIntHashMap;
 import gnu.trove.set.TIntSet;
 import gnu.trove.set.hash.TIntHashSet;
-import nu.marginalia.assistant.api.RpcSimilarDomain;
+import nu.marginalia.api.domains.*;
-import nu.marginalia.assistant.client.model.SimilarDomain;
+import nu.marginalia.api.domains.model.SimilarDomain;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.query.client.QueryClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -27,7 +27,7 @@ public class SimilarDomainsService {
 
     private static final Logger logger = LoggerFactory.getLogger(SimilarDomainsService.class);
     private final HikariDataSource dataSource;
-    private final QueryClient queryClient;
+    private final AggregateDomainLinksClient domainLinksClient;
 
     private volatile TIntIntHashMap domainIdToIdx = new TIntIntHashMap(100_000);
     private volatile int[] domainIdxToId;
@@ -43,9 +43,9 @@ public class SimilarDomainsService {
     volatile boolean isReady = false;
 
     @Inject
-    public SimilarDomainsService(HikariDataSource dataSource, QueryClient queryClient) {
+    public SimilarDomainsService(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) {
         this.dataSource = dataSource;
-        this.queryClient = queryClient;
+        this.domainLinksClient = domainLinksClient;
 
         Executors.newSingleThreadExecutor().submit(this::init);
     }
@@ -256,7 +256,7 @@ public class SimilarDomainsService {
     private TIntSet getLinkingIdsDToS(int domainIdx) {
         var items = new TIntHashSet();
 
-        for (int id : queryClient.getLinksFromDomain(domainIdxToId[domainIdx])) {
+        for (int id : domainLinksClient.getLinksFromDomain(domainIdxToId[domainIdx])) {
             items.add(domainIdToIdx.get(id));
         }
 
@@ -266,7 +266,7 @@ public class SimilarDomainsService {
     private TIntSet getLinkingIdsSToD(int domainIdx) {
         var items = new TIntHashSet();
 
-        for (int id : queryClient.getLinksToDomain(domainIdxToId[domainIdx])) {
+        for (int id : domainLinksClient.getLinksToDomain(domainIdxToId[domainIdx])) {
             items.add(domainIdToIdx.get(id));
         }
 

code/functions/domain-links/aggregate/build.gradle

@@ -0,0 +1,36 @@
+plugins {
+    id 'java'
+
+    id 'application'
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
+}
+
+dependencies {
+    implementation project(':code:functions:domain-links:api')
+
+    implementation project(':code:common:config')
+    implementation project(':code:common:service')
+    implementation project(':code:common:model')
+    implementation project(':code:common:service-discovery')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.bundles.grpc
+    implementation libs.notnull
+    implementation libs.guice
+    implementation libs.fastutil
+    implementation libs.bundles.mariadb
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+
+}

code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java

@@ -0,0 +1,96 @@
+package nu.marginalia.functions.domainlinks;
+
+import com.google.inject.Inject;
+import io.grpc.stub.StreamObserver;
+import nu.marginalia.api.domainlink.*;
+import nu.marginalia.api.indexdomainlinks.PartitionDomainLinksClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase {
+    private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksService.class);
+    private final PartitionDomainLinksClient client;
+
+    @Inject
+    public AggregateDomainLinksService(PartitionDomainLinksClient client) {
+        this.client = client;
+    }
+
+    @Override
+    public void getAllLinks(Empty request,
+                            StreamObserver<RpcDomainIdPairs> responseObserver) {
+
+        client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks)
+                .run(Empty.getDefaultInstance())
+                .forEach(iter -> iter.forEachRemaining(responseObserver::onNext));
+
+        responseObserver.onCompleted();
+    }
+
+    @Override
+    public void getLinksFromDomain(RpcDomainId request,
+                                   StreamObserver<RpcDomainIdList> responseObserver) {
+        var rspBuilder = RpcDomainIdList.newBuilder();
+
+        client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain)
+                .run(request)
+                .stream()
+                .map(RpcDomainIdList::getDomainIdList)
+                .flatMap(List::stream)
+                .forEach(rspBuilder::addDomainId);
+
+        responseObserver.onNext(rspBuilder.build());
+        responseObserver.onCompleted();
+    }
+
+    @Override
+    public void getLinksToDomain(RpcDomainId request,
+                                 StreamObserver<RpcDomainIdList> responseObserver) {
+        var rspBuilder = RpcDomainIdList.newBuilder();
+
+
+        client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain)
+                .run(request)
+                .stream()
+                .map(RpcDomainIdList::getDomainIdList)
+                .flatMap(List::stream)
+                .forEach(rspBuilder::addDomainId);
+
+        responseObserver.onNext(rspBuilder.build());
+        responseObserver.onCompleted();
+    }
+
+    @Override
+    public void countLinksFromDomain(RpcDomainId request,
+                                     StreamObserver<RpcDomainIdCount> responseObserver) {
+        int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain)
+                .run(request)
+                .stream()
+                .mapToInt(RpcDomainIdCount::getIdCount)
+                .sum();
+
+        var rspBuilder = RpcDomainIdCount.newBuilder();
+        rspBuilder.setIdCount(sum);
+        responseObserver.onNext(rspBuilder.build());
+        responseObserver.onCompleted();
+    }
+
+    @Override
+    public void countLinksToDomain(RpcDomainId request,
+                                   StreamObserver<RpcDomainIdCount> responseObserver) {
+
+        int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain)
+                .run(request)
+                .stream()
+                .mapToInt(RpcDomainIdCount::getIdCount)
+                .sum();
+
+        var rspBuilder = RpcDomainIdCount.newBuilder();
+        rspBuilder.setIdCount(sum);
+        responseObserver.onNext(rspBuilder.build());
+        responseObserver.onCompleted();
+    }
+
+}

code/functions/domain-links/api/build.gradle

@@ -0,0 +1,46 @@
+plugins {
+    id 'java'
+
+    id "com.google.protobuf" version "0.9.4"
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
+}
+
+jar.archiveBaseName = 'index-domain-links-api'
+
+sourceSets {
+    main {
+        proto {
+            srcDir 'src/main/protobuf'
+        }
+    }
+}
+
+apply from: "$rootProject.projectDir/protobuf.gradle"
+
+dependencies {
+    implementation project(':code:common:model')
+    implementation project(':code:common:config')
+    implementation project(':code:common:service-discovery')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.notnull
+    implementation libs.guice
+    implementation libs.gson
+    implementation libs.protobuf
+    implementation libs.roaringbitmap
+    implementation libs.javax.annotation
+    implementation libs.bundles.grpc
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+}

code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java

@@ -0,0 +1,138 @@
+package nu.marginalia.api.indexdomainlinks;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.api.domainlink.DomainLinksApiGrpc;
+import nu.marginalia.api.domainlink.Empty;
+import nu.marginalia.api.domainlink.RpcDomainId;
+import nu.marginalia.service.client.GrpcChannelPoolFactory;
+import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
+import org.roaringbitmap.longlong.PeekableLongIterator;
+import org.roaringbitmap.longlong.Roaring64Bitmap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.List;
+
+@Singleton
+public class AggregateDomainLinksClient {
+    private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksClient.class);
+
+    private final GrpcSingleNodeChannelPool<DomainLinksApiGrpc.DomainLinksApiBlockingStub> channelPool;
+
+    @Inject
+    public AggregateDomainLinksClient(GrpcChannelPoolFactory factory) {
+        this.channelPool = factory.createSingle(
+                ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.any()),
+                DomainLinksApiGrpc::newBlockingStub);
+    }
+
+
+    public AllLinks getAllDomainLinks() {
+        AllLinks links = new AllLinks();
+
+        channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks)
+                .run(Empty.getDefaultInstance())
+                .forEachRemaining(pairs -> {
+                    for (int i = 0; i < pairs.getDestIdsCount(); i++) {
+                        links.add(pairs.getSourceIds(i), pairs.getDestIds(i));
+                    }
+                });
+
+        return links;
+    }
+
+    public List<Integer> getLinksToDomain(int domainId) {
+        try {
+            return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain)
+                    .run(RpcDomainId.newBuilder().setDomainId(domainId).build())
+                    .getDomainIdList()
+                    .stream()
+                    .sorted()
+                    .toList();
+        }
+        catch (Exception e) {
+            logger.error("API Exception", e);
+            return List.of();
+        }
+    }
+
+    public List<Integer> getLinksFromDomain(int domainId) {
+        try {
+            return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain)
+                    .run(RpcDomainId.newBuilder().setDomainId(domainId).build())
+                    .getDomainIdList()
+                    .stream()
+                    .sorted()
+                    .toList();
+
+        }
+        catch (Exception e) {
+            logger.error("API Exception", e);
+            return List.of();
+        }
+    }
+
+    public int countLinksToDomain(int domainId) {
+        try {
+            return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain)
+                    .run(RpcDomainId.newBuilder().setDomainId(domainId).build())
+                    .getIdCount();
+
+        }
+        catch (Exception e) {
+            logger.error("API Exception", e);
+            return 0;
+        }
+    }
+
+    public int countLinksFromDomain(int domainId) {
+        try {
+            return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain)
+                    .run(RpcDomainId.newBuilder().setDomainId(domainId).build())
+                    .getIdCount();
+        }
+        catch (Exception e) {
+            logger.error("API Exception", e);
+            return 0;
+        }
+    }
+
+    public boolean waitReady(Duration duration) throws InterruptedException {
+        return channelPool.awaitChannel(duration);
+    }
+
+    public static class AllLinks {
+        private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap();
+
+        public void add(int source, int dest) {
+            sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest));
+        }
+
+        public Iterator iterator() {
+            return new Iterator();
+        }
+
+        public class Iterator {
+            private final PeekableLongIterator base = sourceToDest.getLongIterator();
+            long val = Long.MIN_VALUE;
+
+            public boolean advance() {
+                if (base.hasNext()) {
+                    val = base.next();
+                    return true;
+                }
+                return false;
+            }
+            public int source() {
+                return (int) (val >>> 32);
+            }
+            public int dest() {
+                return (int) (val & 0xFFFF_FFFFL);
+            }
+        }
+    }
+}

code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java

@@ -0,0 +1,30 @@
+package nu.marginalia.api.indexdomainlinks;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.api.domainlink.DomainLinksApiGrpc;
+import nu.marginalia.service.client.GrpcChannelPoolFactory;
+import nu.marginalia.service.client.GrpcMultiNodeChannelPool;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Singleton
+public class PartitionDomainLinksClient {
+    private static final Logger logger = LoggerFactory.getLogger(PartitionDomainLinksClient.class);
+
+    private final GrpcMultiNodeChannelPool<DomainLinksApiGrpc.DomainLinksApiBlockingStub> channelPool;
+
+    @Inject
+    public PartitionDomainLinksClient(GrpcChannelPoolFactory factory) {
+        this.channelPool = factory.createMulti(
+                ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.multi()),
+                DomainLinksApiGrpc::newBlockingStub);
+    }
+
+    public GrpcMultiNodeChannelPool<DomainLinksApiGrpc.DomainLinksApiBlockingStub> getChannelPool() {
+        return channelPool;
+    }
+
+}

code/functions/domain-links/api/src/main/protobuf/domain-links.proto

@@ -0,0 +1,29 @@
+syntax="proto3";
+package nu.marginalia.api.domainlinks;
+
+option java_package="nu.marginalia.api.domainlink";
+option java_multiple_files=true;
+
+service DomainLinksApi {
+  rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {}
+  rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {}
+  rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {}
+  rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {}
+  rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {}
+}
+
+message RpcDomainId {
+  int32 domainId = 1;
+}
+message RpcDomainIdList {
+  repeated int32 domainId = 1 [packed=true];
+}
+message RpcDomainIdCount {
+  int32 idCount = 1;
+}
+message RpcDomainIdPairs {
+  repeated int32 sourceIds = 1 [packed=true];
+  repeated int32 destIds = 2 [packed=true];
+}
+
+message Empty {}

code/functions/domain-links/partition/build.gradle

@@ -0,0 +1,42 @@
+plugins {
+    id 'java'
+
+    id 'application'
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
+}
+
+dependencies {
+    implementation project(':code:functions:domain-links:api')
+
+    implementation project(':code:common:config')
+    implementation project(':code:common:service')
+    implementation project(':code:common:model')
+    implementation project(':code:common:linkdb')
+    implementation project(':code:common:db')
+    implementation project(':code:common:service-discovery')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.bundles.grpc
+    implementation libs.notnull
+    implementation libs.guice
+    implementation libs.spark
+    implementation libs.opencsv
+    implementation libs.trove
+    implementation libs.fastutil
+    implementation libs.bundles.gson
+    implementation libs.bundles.mariadb
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+
+
+}

code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java

@@ -1,22 +1,23 @@
-package nu.marginalia.index.svc;
+package nu.marginalia.functions.domainlinks;
 
 import com.google.inject.Inject;
 import io.grpc.stub.StreamObserver;
-import nu.marginalia.index.api.*;
+import nu.marginalia.api.domainlink.Empty;
+import nu.marginalia.api.domainlink.*;
 import nu.marginalia.linkdb.dlinks.DomainLinkDb;
 
 /**  GRPC service for interrogating domain links
  */
-public class IndexDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase {
+public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase {
     private final DomainLinkDb domainLinkDb;
 
     @Inject
-    public IndexDomainLinksService(DomainLinkDb domainLinkDb) {
+    public PartitionDomainLinksService(DomainLinkDb domainLinkDb) {
         this.domainLinkDb = domainLinkDb;
     }
 
-    public void getAllLinks(nu.marginalia.index.api.Empty request,
+    public void getAllLinks(Empty request,
-                            io.grpc.stub.StreamObserver<nu.marginalia.index.api.RpcDomainIdPairs> responseObserver) {
+                            io.grpc.stub.StreamObserver<RpcDomainIdPairs> responseObserver) {
 
         try (var idsConverter = new AllIdsResponseConverter(responseObserver)) {
             domainLinkDb.forEach(idsConverter::accept);

code/functions/math/api/build.gradle

@@ -11,6 +11,8 @@ java {
     }
 }
 
+jar.archiveBaseName = 'math-api'
+
 sourceSets {
     main {
         proto {

code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java

@@ -0,0 +1,90 @@
+package nu.marginalia.api.math;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.service.client.GrpcChannelPoolFactory;
+import nu.marginalia.service.client.GrpcSingleNodeChannelPool;
+import nu.marginalia.service.discovery.property.ServiceKey;
+import nu.marginalia.service.discovery.property.ServicePartition;
+import nu.marginalia.service.id.ServiceId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+import nu.marginalia.api.math.model.*;
+import nu.marginalia.api.math.MathProtobufCodec.*;
+
+
+@Singleton
+public class MathClient {
+    private static final Logger logger = LoggerFactory.getLogger(MathClient.class);
+
+    private final GrpcSingleNodeChannelPool<MathApiGrpc.MathApiBlockingStub> channelPool;
+    private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor();
+    @Inject
+    public MathClient(GrpcChannelPoolFactory factory) {
+        this.channelPool = factory.createSingle(
+                ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()),
+                MathApiGrpc::newBlockingStub);
+
+    }
+
+    public Future<DictionaryResponse> dictionaryLookup(String word) {
+        return channelPool.call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup)
+                .async(virtualExecutorService)
+                .run(DictionaryLookup.createRequest(word))
+                .thenApply(DictionaryLookup::convertResponse);
+    }
+
+    @SuppressWarnings("unchecked")
+    public Future<List<String>> spellCheck(String word) {
+        return channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck)
+                .async(virtualExecutorService)
+                .run(SpellCheck.createRequest(word))
+                .thenApply(SpellCheck::convertResponse);
+    }
+
+    public Map<String, List<String>> spellCheck(List<String> words, Duration timeout) throws InterruptedException {
+        List<RpcSpellCheckRequest> requests = words.stream().map(SpellCheck::createRequest).toList();
+
+        var future = channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck)
+                .async(virtualExecutorService)
+                .runFor(requests);
+
+        try {
+            var results = future.get();
+            Map<String, List<String>> map = new HashMap<>();
+            for (int i = 0; i < words.size(); i++) {
+                map.put(words.get(i), SpellCheck.convertResponse(results.get(i)));
+            }
+            return map;
+        }
+        catch (ExecutionException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public Future<String> unitConversion(String value, String from, String to) {
+        return channelPool.call(MathApiGrpc.MathApiBlockingStub::unitConversion)
+                .async(virtualExecutorService)
+                .run(UnitConversion.createRequest(from, to, value))
+                .thenApply(UnitConversion::convertResponse);
+    }
+
+    public Future<String> evalMath(String expression) {
+        return channelPool.call(MathApiGrpc.MathApiBlockingStub::evalMath)
+                .async(virtualExecutorService)
+                .run(EvalMath.createRequest(expression))
+                .thenApply(EvalMath::convertResponse);
+    }
+
+    public boolean isAccepting() {
+        return channelPool.hasChannel();
+    }
+}

code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java

@@ -0,0 +1,66 @@
+package nu.marginalia.api.math;
+
+import nu.marginalia.api.math.model.DictionaryEntry;
+import nu.marginalia.api.math.model.DictionaryResponse;
+
+import java.util.List;
+
+public class MathProtobufCodec {
+
+    public static class DictionaryLookup {
+        public static RpcDictionaryLookupRequest createRequest(String word) {
+            return RpcDictionaryLookupRequest.newBuilder()
+                    .setWord(word)
+                    .build();
+        }
+        public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) {
+            return new DictionaryResponse(
+                    rsp.getWord(),
+                    rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList()
+            );
+        }
+
+        private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) {
+            return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition());
+        }
+    }
+
+    public static class SpellCheck {
+        public static RpcSpellCheckRequest createRequest(String text) {
+            return RpcSpellCheckRequest.newBuilder()
+                    .setText(text)
+                    .build();
+        }
+
+        public static List<String> convertResponse(RpcSpellCheckResponse rsp) {
+            return rsp.getSuggestionsList();
+        }
+    }
+
+    public static class UnitConversion {
+        public static RpcUnitConversionRequest createRequest(String from, String to, String unit) {
+            return RpcUnitConversionRequest.newBuilder()
+                    .setFrom(from)
+                    .setTo(to)
+                    .setUnit(unit)
+                    .build();
+        }
+
+        public static String convertResponse(RpcUnitConversionResponse rsp) {
+            return rsp.getResult();
+        }
+    }
+
+    public static class EvalMath {
+        public static RpcEvalMathRequest createRequest(String expression) {
+            return RpcEvalMathRequest.newBuilder()
+                    .setExpression(expression)
+                    .build();
+        }
+
+        public static String convertResponse(RpcEvalMathResponse rsp) {
+            return rsp.getResult();
+        }
+    }
+
+}

code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.client.model;
+package nu.marginalia.api.math.model;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;

code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.client.model;
+package nu.marginalia.api.math.model;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;

code/functions/math/api/src/main/protobuf/math-api.proto

@@ -0,0 +1,57 @@
+syntax="proto3";
+package nu.marginalia.api.math;
+
+option java_package="nu.marginalia.api.math";
+option java_multiple_files=true;
+
+service MathApi {
+  /** Looks up a word in the dictionary. */
+  rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {}
+  /** Checks the spelling of a text. */
+  rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {}
+  /** Converts a unit from one to another. */
+  rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {}
+  /** Evaluates a mathematical expression. */
+  rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {}
+}
+
+message RpcDictionaryLookupRequest {
+  string word = 1;
+}
+
+message RpcDictionaryLookupResponse {
+  string word = 1;
+  repeated RpcDictionaryEntry entries = 2;
+}
+
+message RpcDictionaryEntry {
+  string type = 1;
+  string word = 2;
+  string definition = 3;
+}
+
+message RpcSpellCheckRequest {
+  string text = 1;
+}
+
+message RpcSpellCheckResponse {
+  repeated string suggestions = 1;
+}
+
+message RpcUnitConversionRequest {
+  string unit = 1;
+  string from = 2;
+  string to = 3;
+}
+
+message RpcUnitConversionResponse {
+  string result = 1;
+}
+
+message RpcEvalMathRequest {
+  string expression = 1;
+}
+
+message RpcEvalMathResponse {
+  string result = 1;
+}

code/functions/math/build.gradle

@@ -0,0 +1,32 @@
+plugins {
+    id 'java'
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
+}
+
+dependencies {
+    implementation project(':third-party:symspell')
+    implementation project(':code:functions:math:api')
+
+    implementation libs.bundles.slf4j
+
+    implementation libs.prometheus
+    implementation libs.bundles.grpc
+    implementation libs.notnull
+    implementation libs.guice
+    implementation libs.spark
+    implementation libs.opencsv
+    implementation libs.trove
+    implementation libs.fastutil
+    implementation libs.bundles.gson
+    implementation libs.bundles.mariadb
+
+    testImplementation libs.bundles.slf4j.test
+    testImplementation libs.bundles.junit
+    testImplementation libs.mockito
+}

code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java

@@ -1,34 +1,28 @@
-package nu.marginalia.assistant;
+package nu.marginalia.functions.math;
 
 import com.google.inject.Inject;
 import io.grpc.stub.StreamObserver;
-import nu.marginalia.assistant.api.*;
+import nu.marginalia.api.math.*;
-import nu.marginalia.assistant.dict.DictionaryService;
+import nu.marginalia.functions.math.dict.DictionaryService;
-import nu.marginalia.assistant.dict.SpellChecker;
+import nu.marginalia.functions.math.dict.SpellChecker;
-import nu.marginalia.assistant.domains.DomainInformationService;
+import nu.marginalia.functions.math.eval.MathParser;
-import nu.marginalia.assistant.domains.SimilarDomainsService;
+import nu.marginalia.functions.math.eval.Units;
-import nu.marginalia.assistant.eval.MathParser;
-import nu.marginalia.assistant.eval.Units;
 
-public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase {
+public class MathGrpcService extends MathApiGrpc.MathApiImplBase {
 
     private final DictionaryService dictionaryService;
     private final SpellChecker spellChecker;
     private final Units units;
     private final MathParser mathParser;
-    private final DomainInformationService domainInformationService;
+
-    private final SimilarDomainsService similarDomainsService;
     @Inject
-    public AssistantGrpcService(DictionaryService dictionaryService,
+    public MathGrpcService(DictionaryService dictionaryService, SpellChecker spellChecker, Units units, MathParser mathParser)
-                                SpellChecker spellChecker, Units units, MathParser mathParser, DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService)
     {
 
         this.dictionaryService = dictionaryService;
         this.spellChecker = spellChecker;
         this.units = units;
         this.mathParser = mathParser;
-        this.domainInformationService = domainInformationService;
-        this.similarDomainsService = similarDomainsService;
     }
 
     @Override
@@ -95,37 +89,4 @@ public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase
         responseObserver.onCompleted();
     }
 
-    @Override
-    public void getDomainInfo(RpcDomainId request, StreamObserver<RpcDomainInfoResponse> responseObserver) {
-        var ret = domainInformationService.domainInfo(request.getDomainId());
-
-        ret.ifPresent(responseObserver::onNext);
-
-        responseObserver.onCompleted();
-    }
-
-    @Override
-    public void getSimilarDomains(RpcDomainLinksRequest request,
-                                  StreamObserver<RpcSimilarDomains> responseObserver) {
-        var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount());
-
-        var responseBuilder = RpcSimilarDomains
-                .newBuilder()
-                .addAllDomains(ret);
-
-        responseObserver.onNext(responseBuilder.build());
-        responseObserver.onCompleted();
-    }
-
-    @Override
-    public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver<RpcSimilarDomains> responseObserver) {
-        var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount());
-
-        var responseBuilder = RpcSimilarDomains
-                .newBuilder()
-                .addAllDomains(ret);
-
-        responseObserver.onNext(responseBuilder.build());
-        responseObserver.onCompleted();
-    }
 }

code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java

@@ -1,10 +1,10 @@
-package nu.marginalia.assistant.dict;
+package nu.marginalia.functions.math.dict;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.zaxxer.hikari.HikariDataSource;
-import nu.marginalia.assistant.client.model.DictionaryEntry;
+import nu.marginalia.api.math.model.DictionaryEntry;
-import nu.marginalia.assistant.client.model.DictionaryResponse;
+import nu.marginalia.api.math.model.DictionaryResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.dict;
+package nu.marginalia.functions.math.dict;
 
 import com.google.inject.Singleton;
 import symspell.SymSpell;

code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.eval;
+package nu.marginalia.functions.math.eval;
 
 import lombok.AllArgsConstructor;
 import lombok.SneakyThrows;

code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.eval;
+package nu.marginalia.functions.math.eval;
 
 public class Unit {
 

code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java

@@ -1,4 +1,4 @@
-package nu.marginalia.assistant.eval;
+package nu.marginalia.functions.math.eval;
 
 import com.opencsv.CSVReader;
 import lombok.SneakyThrows;

code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java

@@ -314,6 +314,10 @@ public class MqPersistence {
      */
     public Collection<MqMessage> pollInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException {
 
+        if (dataSource.isClosed()) {
+            return Collections.emptyList();
+        }
+
         // Mark new messages as claimed
         int expected = markInboxMessages(inboxName, instanceUUID, tick, n);
         if (expected == 0) {
@@ -366,6 +370,10 @@ public class MqPersistence {
      */
     public Collection<MqMessage> pollReplyInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException {
 
+        if (dataSource.isClosed()) {
+            return Collections.emptyList();
+        }
+
         // Mark new messages as claimed
         int expected = markInboxMessages(inboxName, instanceUUID, tick, n);
         if (expected == 0) {

code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java

@@ -76,6 +76,7 @@ public class IndexConstructorMain extends ProcessMainClass {
         // Grace period so we don't rug pull the logger or jdbc
         TimeUnit.SECONDS.sleep(5);
 
+
         System.exit(0);
     }
 

code/processes/website-adjacencies-calculator/build.gradle

@@ -23,7 +23,7 @@ dependencies {
     implementation project(':code:common:process')
     implementation project(':code:common:service-discovery')
     implementation project(':code:common:service')
-    implementation project(':code:api:query-api')
+    implementation project(':code:functions:domain-links:api')
 
     implementation libs.bundles.slf4j
 

code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java

@@ -4,7 +4,7 @@ import gnu.trove.list.TIntList;
 import gnu.trove.list.array.TIntArrayList;
 import gnu.trove.map.hash.TIntObjectHashMap;
 import gnu.trove.set.hash.TIntHashSet;
-import nu.marginalia.query.client.QueryClient;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import org.roaringbitmap.RoaringBitmap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -35,14 +35,15 @@ public class AdjacenciesData {
         return ret;
     }
 
-    public AdjacenciesData(QueryClient queryClient,
+    public AdjacenciesData(AggregateDomainLinksClient linksClient,
                            DomainAliases aliases) {
         logger.info("Loading adjacency data");
 
         Map<Integer, RoaringBitmap> tmpMapDtoS = new HashMap<>(100_000);
 
         int count = 0;
-        var allLinks = queryClient.getAllDomainLinks();
+        var allLinks = linksClient.getAllDomainLinks();
+
         for (var iter = allLinks.iterator();;count++) {
             if (!iter.advance()) {
                 break;

code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java

@@ -4,18 +4,20 @@ import com.google.inject.Guice;
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
 import nu.marginalia.ProcessConfiguration;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import nu.marginalia.db.DbDomainQueries;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.process.control.ProcessHeartbeat;
 import nu.marginalia.process.control.ProcessHeartbeatImpl;
-import nu.marginalia.query.client.QueryClient;
 import nu.marginalia.service.MainClass;
+import nu.marginalia.service.ProcessMainClass;
 import nu.marginalia.service.ServiceDiscoveryModule;
 import nu.marginalia.service.module.DatabaseModule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.sql.SQLException;
+import java.time.Duration;
 import java.util.*;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -24,18 +26,18 @@ import java.util.stream.IntStream;
 
 import static nu.marginalia.adjacencies.SparseBitVector.*;
 
-public class WebsiteAdjacenciesCalculator extends MainClass {
+public class WebsiteAdjacenciesCalculator extends ProcessMainClass {
     private final HikariDataSource dataSource;
     public AdjacenciesData adjacenciesData;
     public DomainAliases domainAliases;
     private static final Logger logger = LoggerFactory.getLogger(WebsiteAdjacenciesCalculator.class);
 
     float[] weights;
-    public WebsiteAdjacenciesCalculator(QueryClient queryClient, HikariDataSource dataSource) throws SQLException {
+    public WebsiteAdjacenciesCalculator(AggregateDomainLinksClient domainLinksClient, HikariDataSource dataSource) throws SQLException {
         this.dataSource = dataSource;
 
         domainAliases = new DomainAliases(dataSource);
-        adjacenciesData = new AdjacenciesData(queryClient, domainAliases);
+        adjacenciesData = new AdjacenciesData(domainLinksClient, domainAliases);
         weights = adjacenciesData.getWeights();
     }
 
@@ -146,16 +148,20 @@ public class WebsiteAdjacenciesCalculator extends MainClass {
     }
 
 
-    public static void main(String[] args) throws SQLException {
+    public static void main(String[] args) throws SQLException, InterruptedException {
         var injector = Guice.createInjector(
                 new DatabaseModule(false),
                 new ServiceDiscoveryModule());
 
 
         var dataSource = injector.getInstance(HikariDataSource.class);
-        var qc = injector.getInstance(QueryClient.class);
+        var lc = injector.getInstance(AggregateDomainLinksClient.class);
 
-        var main = new WebsiteAdjacenciesCalculator(qc, dataSource);
+        if (!lc.waitReady(Duration.ofSeconds(30))) {
+            throw new IllegalStateException("Failed to connect to domain-links");
+        }
+
+        var main = new WebsiteAdjacenciesCalculator(lc, dataSource);
 
         if (args.length == 1 && "load".equals(args[0])) {
             var processHeartbeat = new ProcessHeartbeatImpl(

code/services-application/api-service/build.gradle

@@ -2,8 +2,8 @@ plugins {
     id 'java'
 
     id 'application'
-    id 'com.palantir.docker' version '0.35.0'
     id 'jvm-test-suite'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 java {
@@ -19,7 +19,21 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
+
 
 dependencies {
     implementation project(':code:common:db')

code/services-application/dating-service/build.gradle

@@ -2,8 +2,8 @@ plugins {
     id 'java'
 
     id 'application'
-    id 'com.palantir.docker' version '0.35.0'
     id 'jvm-test-suite'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 application {
@@ -13,7 +13,22 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
+
 
 java {
     toolchain {

code/services-application/explorer-service/build.gradle

@@ -2,8 +2,8 @@ plugins {
     id 'java'
 
     id 'application'
-    id 'com.palantir.docker' version '0.35.0'
     id 'jvm-test-suite'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 application {
@@ -13,7 +13,22 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
+
 
 java {
     toolchain {

code/services-application/search-service/build.gradle

@@ -2,9 +2,25 @@ plugins {
     id 'java'
     id 'io.freefair.sass-base' version '8.4'
     id 'io.freefair.sass-java' version '8.4'
-    id 'com.palantir.docker' version '0.35.0'
     id 'application'
     id 'jvm-test-suite'
+
+    id 'com.google.cloud.tools.jib' version '3.4.0'
+}
+
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
 }
 
 application {
@@ -14,7 +30,6 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
 
 java {
     toolchain {
@@ -26,6 +41,7 @@ sass {
     sourceMapEmbed = true
     outputStyle = EXPANDED
 }
+
 dependencies {
     implementation project(':code:common:db')
     implementation project(':code:common:model')
@@ -38,7 +54,8 @@ dependencies {
     implementation project(':code:libraries:braille-block-punch-cards')
     implementation project(':code:libraries:term-frequency-dict')
 
-    implementation project(':code:api:assistant-api')
+    implementation project(':code:functions:math:api')
+    implementation project(':code:functions:domain-info:api')
     implementation project(':code:api:query-api')
     implementation project(':code:api:index-api')
     implementation project(':code:common:service-discovery')

code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java

@@ -4,7 +4,7 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import lombok.SneakyThrows;
 import nu.marginalia.WebsiteUrl;
-import nu.marginalia.assistant.client.AssistantClient;
+import nu.marginalia.api.math.MathClient;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.db.DbDomainQueries;
 import nu.marginalia.query.client.QueryClient;
@@ -34,7 +34,7 @@ public class SearchOperator {
     // Marker for filtering out sensitive content from the persistent logs
     private final Marker queryMarker = MarkerFactory.getMarker("QUERY");
 
-    private final AssistantClient assistantClient;
+    private final MathClient mathClient;
     private final DbDomainQueries domainQueries;
     private final QueryClient queryClient;
     private final SearchQueryIndexService searchQueryService;
@@ -44,7 +44,7 @@ public class SearchOperator {
 
 
     @Inject
-    public SearchOperator(AssistantClient assistantClient,
+    public SearchOperator(MathClient mathClient,
                           DbDomainQueries domainQueries,
                           QueryClient queryClient,
                           SearchQueryIndexService searchQueryService,
@@ -53,7 +53,7 @@ public class SearchOperator {
                           SearchUnitConversionService searchUnitConversionService)
     {
 
-        this.assistantClient = assistantClient;
+        this.mathClient = mathClient;
         this.domainQueries = domainQueries;
         this.queryClient = queryClient;
 
@@ -162,7 +162,7 @@ public class SearchOperator {
 
     @SneakyThrows
     private void spellCheckTerms(QueryResponse response) {
-        var suggestions = assistantClient
+        var suggestions = mathClient
                 .spellCheck(response.searchTermsHuman(), Duration.ofMillis(20));
 
         suggestions.entrySet()

code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java

@@ -2,11 +2,11 @@
 package nu.marginalia.search.command.commands;
 
 import com.google.inject.Inject;
-import nu.marginalia.assistant.client.AssistantClient;
+import nu.marginalia.api.math.MathClient;
-import nu.marginalia.assistant.client.model.DictionaryResponse;
+import nu.marginalia.api.math.model.DictionaryResponse;
+import nu.marginalia.renderer.MustacheRenderer;
 import nu.marginalia.search.command.SearchCommandInterface;
 import nu.marginalia.search.command.SearchParameters;
-import nu.marginalia.renderer.MustacheRenderer;
 import nu.marginalia.renderer.RendererFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,18 +23,18 @@ public class DefinitionCommand implements SearchCommandInterface {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     private final MustacheRenderer<DictionaryResponse> dictionaryRenderer;
-    private final AssistantClient assistantClient;
+    private final MathClient mathClient;
 
 
     private final Predicate<String> queryPatternPredicate = Pattern.compile("^define:[A-Za-z\\s-0-9]+$").asPredicate();
 
     @Inject
-    public DefinitionCommand(RendererFactory rendererFactory, AssistantClient assistantClient)
+    public DefinitionCommand(RendererFactory rendererFactory, MathClient mathClient)
             throws IOException
     {
 
         dictionaryRenderer = rendererFactory.renderer("search/dictionary-results");
-        this.assistantClient = assistantClient;
+        this.mathClient = mathClient;
     }
 
     @Override
@@ -57,9 +57,9 @@ public class DefinitionCommand implements SearchCommandInterface {
         String word = humanQuery.substring(definePrefix.length()).toLowerCase();
 
         try {
-            return assistantClient
+            return mathClient
                     .dictionaryLookup(word)
-                    .get(100, TimeUnit.MILLISECONDS);
+                    .get(250, TimeUnit.MILLISECONDS);
         }
         catch (Exception e) {
             logger.error("Failed to lookup definition for word: " + word, e);

code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java

@@ -14,17 +14,13 @@ import java.io.IOException;
 import java.util.Optional;
 
 public class SearchCommand implements SearchCommandInterface {
-    private final DomainBlacklist blacklist;
     private final SearchOperator searchOperator;
     private final MustacheRenderer<DecoratedSearchResults> searchResultsRenderer;
 
 
     @Inject
-    public SearchCommand(DomainBlacklist blacklist,
+    public SearchCommand(SearchOperator searchOperator,
-                         SearchOperator searchOperator,
+                         RendererFactory rendererFactory) throws IOException {
-                         RendererFactory rendererFactory
-                         ) throws IOException {
-        this.blacklist = blacklist;
         this.searchOperator = searchOperator;
 
         searchResultsRenderer = rendererFactory.renderer("search/search-results");

code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java

@@ -1,8 +1,8 @@
 package nu.marginalia.search.svc;
 
 import com.google.inject.Inject;
-import nu.marginalia.assistant.client.AssistantClient;
+import nu.marginalia.api.domains.DomainInfoClient;
-import nu.marginalia.assistant.client.model.SimilarDomain;
+import nu.marginalia.api.domains.model.SimilarDomain;
 import nu.marginalia.browse.DbBrowseDomainsRandom;
 import nu.marginalia.browse.model.BrowseResult;
 import nu.marginalia.browse.model.BrowseResultSet;
@@ -25,20 +25,20 @@ public class SearchBrowseService {
     private final DbBrowseDomainsRandom randomDomains;
     private final DbDomainQueries domainQueries;
     private final DomainBlacklist blacklist;
-    private final AssistantClient assistantClient;
+    private final DomainInfoClient domainInfoClient;
     private final BrowseResultCleaner browseResultCleaner;
 
     @Inject
     public SearchBrowseService(DbBrowseDomainsRandom randomDomains,
                                DbDomainQueries domainQueries,
                                DomainBlacklist blacklist,
-                               AssistantClient assistantClient,
+                               DomainInfoClient domainInfoClient,
                                BrowseResultCleaner browseResultCleaner)
     {
         this.randomDomains = randomDomains;
         this.domainQueries = domainQueries;
         this.blacklist = blacklist;
-        this.assistantClient = assistantClient;
+        this.domainInfoClient = domainInfoClient;
         this.browseResultCleaner = browseResultCleaner;
     }
 
@@ -53,7 +53,7 @@ public class SearchBrowseService {
     public BrowseResultSet getRelatedEntries(String domainName) throws ExecutionException, InterruptedException, TimeoutException {
         var domain = domainQueries.getDomainId(new EdgeDomain(domainName));
 
-        var neighbors = assistantClient.similarDomains(domain, 50)
+        var neighbors = domainInfoClient.similarDomains(domain, 50)
                 .get(100, TimeUnit.MILLISECONDS);
 
         neighbors.removeIf(sd -> !sd.screenshot());
@@ -61,7 +61,7 @@ public class SearchBrowseService {
         // If the results are very few, supplement with the alternative shitty algorithm
         if (neighbors.size() < 25) {
             Set<SimilarDomain> allNeighbors = new HashSet<>(neighbors);
-            allNeighbors.addAll(assistantClient
+            allNeighbors.addAll(domainInfoClient
                     .linkedDomains(domain, 50)
                     .get(100, TimeUnit.MILLISECONDS)
             );

code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java

@@ -1,8 +1,9 @@
 package nu.marginalia.search.svc;
 
 import com.google.inject.Inject;
-import nu.marginalia.assistant.client.AssistantClient;
+import nu.marginalia.api.domains.DomainInfoClient;
-import nu.marginalia.assistant.client.model.SimilarDomain;
+import nu.marginalia.api.domains.model.DomainInformation;
+import nu.marginalia.api.domains.model.SimilarDomain;
 import nu.marginalia.db.DbDomainQueries;
 import nu.marginalia.feedlot.model.FeedItems;
 import nu.marginalia.model.EdgeDomain;
@@ -10,7 +11,6 @@ import nu.marginalia.renderer.MustacheRenderer;
 import nu.marginalia.renderer.RendererFactory;
 import nu.marginalia.screenshot.ScreenshotService;
 import nu.marginalia.search.SearchOperator;
-import nu.marginalia.assistant.client.model.DomainInformation;
 import nu.marginalia.feedlot.FeedlotClient;
 import nu.marginalia.search.model.UrlDetails;
 import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData;
@@ -32,7 +32,7 @@ public class SearchSiteInfoService {
     private static final Logger logger = LoggerFactory.getLogger(SearchSiteInfoService.class);
 
     private final SearchOperator searchOperator;
-    private final AssistantClient assistantClient;
+    private final DomainInfoClient domainInfoClient;
     private final SearchFlagSiteService flagSiteService;
     private final DbDomainQueries domainQueries;
     private final MustacheRenderer<Object> renderer;
@@ -41,7 +41,7 @@ public class SearchSiteInfoService {
 
     @Inject
     public SearchSiteInfoService(SearchOperator searchOperator,
-                                 AssistantClient assistantClient,
+                                 DomainInfoClient domainInfoClient,
                                  RendererFactory rendererFactory,
                                  SearchFlagSiteService flagSiteService,
                                  DbDomainQueries domainQueries,
@@ -49,7 +49,7 @@ public class SearchSiteInfoService {
                                  ScreenshotService screenshotService) throws IOException
     {
         this.searchOperator = searchOperator;
-        this.assistantClient = assistantClient;
+        this.domainInfoClient = domainInfoClient;
         this.flagSiteService = flagSiteService;
         this.domainQueries = domainQueries;
 
@@ -137,15 +137,20 @@ public class SearchSiteInfoService {
         boolean hasScreenshot = screenshotService.hasScreenshot(domainId);
 
         var feedItemsFuture = feedlotClient.getFeedItems(domainName);
-        if (domainId < 0 || !assistantClient.isAccepting()) {
+        if (domainId < 0) {
+            domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
+            similarSetFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
+            linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
+        }
+        else if (!domainInfoClient.isAccepting()) {
             domainInfoFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
             similarSetFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
             linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable"));
         }
         else {
-            domainInfoFuture = assistantClient.domainInformation(domainId);
+            domainInfoFuture = domainInfoClient.domainInformation(domainId);
-            similarSetFuture = assistantClient.similarDomains(domainId, 25);
+            similarSetFuture = domainInfoClient.similarDomains(domainId, 25);
-            linkingDomainsFuture = assistantClient.linkedDomains(domainId, 25);
+            linkingDomainsFuture = domainInfoClient.linkedDomains(domainId, 25);
         }
 
         List<UrlDetails> sampleResults = searchOperator.doSiteSearch(domainName, 5);
@@ -174,7 +179,7 @@ public class SearchSiteInfoService {
 
     private <T> T waitForFuture(Future<T> future, Supplier<T> fallback) {
         try {
-            return future.get(50, TimeUnit.MILLISECONDS);
+            return future.get(250, TimeUnit.MILLISECONDS);
         } catch (Exception e) {
             logger.info("Failed to get domain data: {}", e.getMessage());
             return fallback.get();

code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java

@@ -1,6 +1,6 @@
 package nu.marginalia.search.svc;
 
-import nu.marginalia.assistant.client.AssistantClient;
+import nu.marginalia.api.math.MathClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,11 +21,11 @@ public class SearchUnitConversionService {
     private final Pattern conversionPattern = Pattern.compile("((\\d+|\\s+|[.()\\-^+%*/]|log[^a-z]|log2[^a-z]|sqrt[^a-z]|log10|cos[^a-z]|sin[^a-z]|tan[^a-z]|log2|pi[^a-z]|e[^a-z]|2pi[^a-z])+)\\s*([a-zA-Z][a-zA-Z^.0-9]*\\s?[a-zA-Z^.0-9]*)\\s+in\\s+([a-zA-Z^.0-9]+\\s?[a-zA-Z^.0-9]*)");
     private final Predicate<String> evalPredicate = Pattern.compile("(\\d+|\\s+|[.()\\-^+%*/]|log|log2|sqrt|log10|cos|sin|tan|pi|e|2pi)+").asMatchPredicate();
 
-    private final AssistantClient assistantClient;
+    private final MathClient mathClient;
 
     @Inject
-    public SearchUnitConversionService(AssistantClient assistantClient) {
+    public SearchUnitConversionService(MathClient mathClient) {
-        this.assistantClient = assistantClient;
+        this.mathClient = mathClient;
     }
 
     public Optional<String> tryConversion(String query) {
@@ -40,9 +40,9 @@ public class SearchUnitConversionService {
         logger.info("{} -> '{}' '{}' '{}'", query, value, from, to);
 
         try {
-            var resultFuture = assistantClient.unitConversion(value, from, to);
+            var resultFuture = mathClient.unitConversion(value, from, to);
             return Optional.of(
-                    resultFuture.get(100, TimeUnit.MILLISECONDS)
+                    resultFuture.get(250, TimeUnit.MILLISECONDS)
             );
         } catch (ExecutionException e) {
             logger.error("Error in unit conversion", e);
@@ -68,6 +68,6 @@ public class SearchUnitConversionService {
 
         logger.info("eval({})", expr);
 
-        return assistantClient.evalMath(expr);
+        return mathClient.evalMath(expr);
     }
 }

code/services-core/assistant-service/build.gradle

@@ -3,7 +3,7 @@ plugins {
 
     id 'application'
     id 'jvm-test-suite'
-    id 'com.palantir.docker' version '0.35.0'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 application {
@@ -13,7 +13,22 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
+
 
 java {
     toolchain {
@@ -23,7 +38,12 @@ java {
 
 dependencies {
     implementation project(':third-party:symspell')
-    implementation project(':code:api:assistant-api')
+
+    implementation project(':code:functions:math')
+    implementation project(':code:functions:math:api')
+    implementation project(':code:functions:domain-info')
+    implementation project(':code:functions:domain-info:api')
+
     implementation project(':code:api:query-api')
     implementation project(':code:common:config')
     implementation project(':code:common:service')

code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java

@@ -4,8 +4,11 @@ import com.google.gson.Gson;
 import com.google.inject.Inject;
 import lombok.SneakyThrows;
 import nu.marginalia.assistant.suggest.Suggestions;
+import nu.marginalia.functions.domains.DomainInfoGrpcService;
+import nu.marginalia.functions.math.MathGrpcService;
 import nu.marginalia.model.gson.GsonFactory;
 import nu.marginalia.screenshot.ScreenshotService;
+import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.server.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -24,10 +27,13 @@ public class AssistantService extends Service {
     @Inject
     public AssistantService(BaseServiceParams params,
                             ScreenshotService screenshotService,
-                            AssistantGrpcService assistantGrpcService,
+                            DomainInfoGrpcService domainInfoGrpcService,
+                            MathGrpcService mathGrpcService,
                             Suggestions suggestions)
     {
-        super(params, List.of(assistantGrpcService));
+        super(params,
+                ServicePartition.any(),
+                List.of(domainInfoGrpcService, mathGrpcService));
 
         this.suggestions = suggestions;
 

code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java

@@ -2,9 +2,9 @@ package nu.marginalia.assistant.suggest;
 
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
+import nu.marginalia.functions.math.dict.SpellChecker;
 import nu.marginalia.term_frequency_dict.TermFrequencyDict;
 import nu.marginalia.model.crawl.HtmlFeature;
-import nu.marginalia.assistant.dict.SpellChecker;
 import org.apache.commons.collections4.trie.PatriciaTrie;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;

code/services-core/control-service/build.gradle

@@ -1,9 +1,8 @@
 plugins {
     id 'java'
-
     id 'application'
-    id 'com.palantir.docker' version '0.35.0'
     id 'jvm-test-suite'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 java {
@@ -19,7 +18,22 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
+
 
 dependencies {
     implementation libs.bundles.gson

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -60,6 +60,7 @@ public class ControlService extends Service {
                       ) throws IOException {
 
         super(params);
+
         this.monitors = monitors;
         this.heartbeatService = heartbeatService;
         this.eventLogService = eventLogService;

code/services-core/executor-service/build.gradle

@@ -1,9 +1,9 @@
 plugins {
     id 'java'
 
-    id 'com.palantir.docker' version '0.35.0'
     id 'application'
     id 'jvm-test-suite'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 application {
@@ -13,7 +13,24 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service-with-dist.gradle"
+clean {
+    delete fileTree('build/dist-extra')
+}
+
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
 
 java {
     toolchain {
@@ -22,6 +39,15 @@ java {
 }
 
 dependencies {
+    // These look weird but they're needed to be able to spawn the processes
+    // from the executor service
+
+    implementation project(':code:processes:website-adjacencies-calculator')
+    implementation project(':code:processes:crawling-process')
+    implementation project(':code:processes:loading-process')
+    implementation project(':code:processes:converting-process')
+    implementation project(':code:processes:index-constructor-process')
+
     implementation project(':code:common:config')
     implementation project(':code:common:model')
     implementation project(':code:common:process')
@@ -35,6 +61,8 @@ dependencies {
 
     implementation project(':code:libraries:message-queue')
 
+    implementation project(':code:functions:domain-links:api')
+
     implementation project(':code:process-models:crawl-spec')
     implementation project(':code:process-models:crawling-model')
     implementation project(':code:features-crawl:link-parser')

code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java

@@ -6,6 +6,7 @@ import com.google.inject.Singleton;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.actor.prototype.RecordActorPrototype;
 import nu.marginalia.actor.state.ActorStep;
+import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient;
 import nu.marginalia.query.client.QueryClient;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.storage.model.FileStorageId;
@@ -32,7 +33,7 @@ public class ExportDataActor extends RecordActorPrototype {
     private final FileStorageService storageService;
     private final HikariDataSource dataSource;
     private final Logger logger = LoggerFactory.getLogger(getClass());
-    private final QueryClient queryClient;
+    private final AggregateDomainLinksClient domainLinksClient;
 
     public record Export() implements ActorStep {}
     public record ExportBlacklist(FileStorageId fid) implements ActorStep {}
@@ -114,7 +115,7 @@ public class ExportDataActor extends RecordActorPrototype {
                 var tmpFile = Files.createTempFile(storage.asPath(), "export", ".csv.gz",
                         PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")));
 
-                var allLinks = queryClient.getAllDomainLinks();
+                var allLinks = domainLinksClient.getAllDomainLinks();
 
                 try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)))))
                 {
@@ -154,12 +155,13 @@ public class ExportDataActor extends RecordActorPrototype {
     @Inject
     public ExportDataActor(Gson gson,
                            FileStorageService storageService,
-                           HikariDataSource dataSource, QueryClient queryClient)
+                           HikariDataSource dataSource,
+                           AggregateDomainLinksClient domainLinksClient)
     {
         super(gson);
         this.storageService = storageService;
         this.dataSource = dataSource;
-        this.queryClient = queryClient;
+        this.domainLinksClient = domainLinksClient;
     }
 
 }

code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java

@@ -2,12 +2,14 @@ package nu.marginalia.executor;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.name.Names;
+import nu.marginalia.WmsaHome;
 
 import java.nio.file.Path;
 
 public class ExecutorModule extends AbstractModule  {
     public void configure() {
-        String dist = System.getProperty("distPath", System.getProperty("WMSA_HOME", "/var/lib/wmsa") + "/dist/current");
+
+        String dist = System.getProperty("distPath", WmsaHome.getHomePath().resolve("/dist").toString());
         bind(Path.class).annotatedWith(Names.named("distPath")).toInstance(Path.of(dist));
     }
 }

code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java

@@ -1,10 +1,10 @@
 package nu.marginalia.executor;
 
-import com.google.gson.Gson;
 import com.google.inject.Inject;
 import nu.marginalia.actor.ExecutorActor;
 import nu.marginalia.actor.ExecutorActorControlService;
 import nu.marginalia.executor.svc.TransferService;
+import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.service.server.BaseServiceParams;
 import nu.marginalia.service.server.Service;
 import nu.marginalia.service.server.mq.MqRequest;
@@ -16,10 +16,7 @@ import java.util.List;
 
 // Weird name for this one to not have clashes with java.util.concurrent.ExecutorService
 public class ExecutorSvc extends Service {
-    private final BaseServiceParams params;
-    private final Gson gson;
     private final ExecutorActorControlService actorControlService;
-    private final TransferService transferService;
 
     private static final Logger logger = LoggerFactory.getLogger(ExecutorSvc.class);
 
@@ -28,14 +25,12 @@ public class ExecutorSvc extends Service {
     public ExecutorSvc(BaseServiceParams params,
                        ExecutorActorControlService actorControlService,
                        ExecutorGrpcService executorGrpcService,
-                       Gson gson,
                        TransferService transferService)
     {
-        super(params, List.of(executorGrpcService));
+        super(params,
-        this.params = params;
+                ServicePartition.partition(params.configuration.node()),
-        this.gson = gson;
+                List.of(executorGrpcService));
         this.actorControlService = actorControlService;
-        this.transferService = transferService;
 
         Spark.get("/transfer/file/:fid", transferService::transferFile);
     }

code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java

@@ -3,6 +3,14 @@ package nu.marginalia.process;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
+import nu.marginalia.WmsaHome;
+import nu.marginalia.adjacencies.WebsiteAdjacenciesCalculator;
+import nu.marginalia.converting.ConverterMain;
+import nu.marginalia.crawl.CrawlerMain;
+import nu.marginalia.index.IndexConstructorMain;
+import nu.marginalia.loading.LoaderMain;
+import nu.marginalia.service.MainClass;
+import nu.marginalia.service.ProcessMainClass;
 import nu.marginalia.service.control.ServiceEventLog;
 import nu.marginalia.service.server.BaseServiceParams;
 import org.slf4j.Logger;
@@ -43,16 +51,32 @@ public class ProcessService {
     }
 
     public enum ProcessId {
-        CRAWLER("crawler-process/bin/crawler-process"),
+        CRAWLER(CrawlerMain.class),
-        CONVERTER("converter-process/bin/converter-process"),
+        CONVERTER(ConverterMain.class),
-        LOADER("loader-process/bin/loader-process"),
+        LOADER(LoaderMain.class),
-        INDEX_CONSTRUCTOR("index-construction-process/bin/index-construction-process"),
+        INDEX_CONSTRUCTOR(IndexConstructorMain.class),
-        ADJACENCIES_CALCULATOR("website-adjacencies-calculator/bin/website-adjacencies-calculator")
+        ADJACENCIES_CALCULATOR(WebsiteAdjacenciesCalculator.class)
         ;
 
-        public final String path;
+        public final String mainClass;
-        ProcessId(String path) {
+        ProcessId(Class<? extends ProcessMainClass> mainClass) {
-            this.path = path;
+            this.mainClass = mainClass.getName();
+        }
+
+        List<String> envOpts() {
+            String variable = switch (this) {
+                case CRAWLER -> "CRAWLER_PROCESS_OPTS";
+                case CONVERTER -> "CONVERTER_PROCESS_OPTS";
+                case LOADER -> "LOADER_PROCESS_OPTS";
+                case INDEX_CONSTRUCTOR -> "INDEX_CONSTRUCTION_PROCESS_OPTS";
+                case ADJACENCIES_CALCULATOR -> "ADJACENCIES_CALCULATOR_PROCESS_OPTS";
+            };
+            String value = System.getenv(variable);
+
+            if (value == null)
+                return List.of();
+            else
+                return Arrays.asList(value.split("\\s+"));
         }
     }
 
@@ -63,27 +87,27 @@ public class ProcessService {
         this.distPath = distPath;
     }
 
-    public boolean trigger(ProcessId processId) throws Exception {
-        return trigger(processId, new String[0]);
-    }
 
-    public boolean trigger(ProcessId processId, String... parameters) throws Exception {
+    public boolean trigger(ProcessId processId, String... extraArgs) throws Exception {
-        final String processPath = distPath.resolve(processId.path).toString();
         final String[] env = createEnvironmentVariables();
-        final String[] args = createCommandArguments(processPath, parameters);
+        List<String> args = new ArrayList<>();
+        String javaHome = System.getProperty("java.home");
+
+        args.add(STR."\{javaHome}/bin/java");
+        args.add("-cp");
+        args.add(System.getProperty("java.class.path"));
+        args.add("--enable-preview");
+        args.addAll(processId.envOpts());
+        args.add(processId.mainClass);
+        args.addAll(Arrays.asList(extraArgs));
 
         Process process;
 
-        if (!Files.exists(Path.of(processPath))) {
+        logger.info("Starting process: {} {}", processId, processId.envOpts());
-            logger.error("Process not found: {}", processPath);
-            return false;
-        }
-
-        logger.info("Starting process: {}: {} // {}", processId, Arrays.toString(args), Arrays.toString(env));
 
         synchronized (processes) {
             if (processes.containsKey(processId)) return false;
-            process = Runtime.getRuntime().exec(args, env);
+            process = Runtime.getRuntime().exec(args.toArray(String[]::new), env);
             processes.put(processId, process);
         }
 
@@ -107,13 +131,6 @@ public class ProcessService {
     }
 
 
-    private String[] createCommandArguments(String processPath, String[] parameters) {
-        final String[] args = new String[parameters.length + 1];
-        args[0] = processPath;
-        System.arraycopy(parameters, 0, args, 1, parameters.length);
-        return args;
-    }
-
     public boolean isRunning(ProcessId processId) {
         return processes.containsKey(processId);
     }
@@ -131,25 +148,14 @@ public class ProcessService {
     /** These environment variables are propagated from the parent process to the child process,
      * along with WMSA_HOME, but it has special logic */
     private final List<String> propagatedEnvironmentVariables = List.of(
-            "JAVA_HOME",
             "ZOOKEEPER_HOSTS",
-            "WMSA_SERVICE_NODE",
+            "WMSA_SERVICE_NODE"
-            "CONVERTER_PROCESS_OPTS",
+    );
-            "LOADER_PROCESS_OPTS",
-            "INDEX_CONSTRUCTION_PROCESS_OPTS",
-            "CRAWLER_PROCESS_OPTS");
 
     private String[] createEnvironmentVariables() {
         List<String> opts = new ArrayList<>();
 
-        String WMSA_HOME = System.getenv("WMSA_HOME");
+        opts.add(env2str("WMSA_HOME", WmsaHome.getHomePath().toString()));
-
-        if (WMSA_HOME == null || WMSA_HOME.isBlank()) {
-            WMSA_HOME = "/var/lib/wmsa";
-        }
-
-        opts.add(env2str("WMSA_HOME", WMSA_HOME));
-        opts.add(env2str("JAVA_OPTS", "--enable-preview")); //
 
         for (String envKey : propagatedEnvironmentVariables) {
             String envValue = System.getenv(envKey);

code/services-core/index-service/build.gradle

@@ -1,9 +1,9 @@
 plugins {
     id 'java'
 
-    id 'com.palantir.docker' version '0.35.0'
     id 'application'
     id 'jvm-test-suite'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 application {
@@ -13,7 +13,22 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
+
 
 java {
     toolchain {
@@ -25,8 +40,13 @@ dependencies {
     implementation project(':code:common:model')
     implementation project(':code:common:db')
     implementation project(':code:common:linkdb')
+
+    implementation project(':code:functions:domain-links:partition')
+    implementation project(':code:functions:domain-links:api')
+
     implementation project(':code:common:service')
     implementation project(':code:api:index-api')
+
     implementation project(':code:common:service-discovery')
 
     implementation project(':code:libraries:array')

code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java

@@ -4,8 +4,9 @@ import com.google.gson.Gson;
 import com.google.inject.Inject;
 import lombok.SneakyThrows;
 import nu.marginalia.IndexLocations;
-import nu.marginalia.index.svc.IndexDomainLinksService;
+import nu.marginalia.functions.domainlinks.PartitionDomainLinksService;
 import nu.marginalia.linkdb.dlinks.DomainLinkDb;
+import nu.marginalia.service.discovery.property.ServicePartition;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.index.client.IndexMqEndpoints;
 import nu.marginalia.index.index.SearchIndex;
@@ -51,10 +52,14 @@ public class IndexService extends Service {
                         FileStorageService fileStorageService,
                         DocumentDbReader documentDbReader,
                         DomainLinkDb domainLinkDb,
-                        IndexDomainLinksService indexDomainLinksService,
+
+                        PartitionDomainLinksService partitionDomainLinksService,
+
                         ServiceEventLog eventLog)
     {
-        super(params, List.of(indexQueryService, indexDomainLinksService));
+        super(params,
+                ServicePartition.partition(params.configuration.node()),
+                List.of(indexQueryService, partitionDomainLinksService));
 
         this.opsService = opsService;
         this.searchIndex = searchIndex;

code/services-core/query-service/build.gradle

@@ -1,9 +1,9 @@
 plugins {
     id 'java'
 
-    id 'com.palantir.docker' version '0.35.0'
     id 'application'
     id 'jvm-test-suite'
+    id 'com.google.cloud.tools.jib' version '3.4.0'
 }
 
 application {
@@ -13,7 +13,22 @@ application {
 
 tasks.distZip.enabled = false
 
-apply from: "$rootProject.projectDir/docker-service.gradle"
+jib {
+    from {
+        image = image = rootProject.ext.dockerImageBase
+    }
+    to {
+        image = 'marginalia/'+project.name
+        tags = ['latest']
+    }
+    container {
+
+        mainClass = application.mainClass
+        jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa']
+        volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa']
+    }
+}
+
 
 java {
     toolchain {
@@ -35,6 +50,9 @@ dependencies {
     implementation project(':code:libraries:language-processing')
     implementation project(':code:libraries:term-frequency-dict')
 
+    implementation project(':code:functions:domain-links:api')
+    implementation project(':code:functions:domain-links:aggregate')
+
     implementation libs.bundles.slf4j
 
     implementation libs.spark

code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java

@@ -1,90 +0,0 @@
-package nu.marginalia.query;
-
-import com.google.inject.Inject;
-import io.grpc.stub.StreamObserver;
-import nu.marginalia.service.client.GrpcMultiNodeChannelPool;
-import nu.marginalia.index.api.IndexDomainLinksApiGrpc;
-import nu.marginalia.index.api.RpcDomainIdCount;
-import nu.marginalia.index.api.RpcDomainIdList;
-import nu.marginalia.index.api.RpcDomainIdPairs;
-import nu.marginalia.service.client.GrpcChannelPoolFactory;
-import nu.marginalia.service.id.ServiceId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class QueryGRPCDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase {
-    private static final Logger logger = LoggerFactory.getLogger(QueryGRPCDomainLinksService.class);
-    private final GrpcMultiNodeChannelPool<IndexDomainLinksApiGrpc.IndexDomainLinksApiBlockingStub> channelPool;
-
-    @Inject
-    public QueryGRPCDomainLinksService(GrpcChannelPoolFactory channelPoolFactory) {
-        channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexDomainLinksApiGrpc::newBlockingStub);
-    }
-
-    @Override
-    public void getAllLinks(nu.marginalia.index.api.Empty request,
-                            StreamObserver<RpcDomainIdPairs> responseObserver) {
-        channelPool.callEachSequential(stub -> stub.getAllLinks(request))
-                .forEach(
-                        iter -> iter.forEachRemaining(responseObserver::onNext)
-                );
-
-        responseObserver.onCompleted();
-    }
-
-    @Override
-    public void getLinksFromDomain(nu.marginalia.index.api.RpcDomainId request,
-                                   StreamObserver<RpcDomainIdList> responseObserver) {
-        var rspBuilder = RpcDomainIdList.newBuilder();
-
-        channelPool.callEachSequential(stub -> stub.getLinksFromDomain(request))
-                .map(RpcDomainIdList::getDomainIdList)
-                .forEach(rspBuilder::addAllDomainId);
-
-        responseObserver.onNext(rspBuilder.build());
-        responseObserver.onCompleted();
-    }
-
-    @Override
-    public void getLinksToDomain(nu.marginalia.index.api.RpcDomainId request,
-                                 StreamObserver<RpcDomainIdList> responseObserver) {
-        var rspBuilder = RpcDomainIdList.newBuilder();
-
-        channelPool.callEachSequential(stub -> stub.getLinksToDomain(request))
-                .map(RpcDomainIdList::getDomainIdList)
-                .forEach(rspBuilder::addAllDomainId);
-
-        responseObserver.onNext(rspBuilder.build());
-        responseObserver.onCompleted();
-    }
-
-    @Override
-    public void countLinksFromDomain(nu.marginalia.index.api.RpcDomainId request,
-                                     StreamObserver<RpcDomainIdCount> responseObserver) {
-
-        int sum = channelPool.callEachSequential(stub -> stub.countLinksFromDomain(request))
-                .mapToInt(RpcDomainIdCount::getIdCount)
-                .sum();
-
-        var rspBuilder = RpcDomainIdCount.newBuilder();
-        rspBuilder.setIdCount(sum);
-        responseObserver.onNext(rspBuilder.build());
-        responseObserver.onCompleted();
-    }
-
-    @Override
-    public void countLinksToDomain(nu.marginalia.index.api.RpcDomainId request,
-                                   io.grpc.stub.StreamObserver<nu.marginalia.index.api.RpcDomainIdCount> responseObserver) {
-
-        int sum = channelPool.callEachSequential(stub -> stub.countLinksToDomain(request))
-                .mapToInt(RpcDomainIdCount::getIdCount)
-                .sum();
-
-        var rspBuilder = RpcDomainIdCount.newBuilder();
-        rspBuilder.setIdCount(sum);
-        responseObserver.onNext(rspBuilder.build());
-        responseObserver.onCompleted();
-    }
-
-}