diff --git a/build.gradle b/build.gradle index 9eb4ef2f..12ab5b31 100644 --- a/build.gradle +++ b/build.gradle @@ -3,6 +3,10 @@ plugins { id("org.jetbrains.gradle.plugin.idea-ext") version "1.0" id "io.freefair.lombok" version "8.3" id "me.champeau.jmh" version "0.6.6" + + // This is a workaround for a bug in the Jib plugin that causes it to stall randomly + // https://github.com/GoogleContainerTools/jib/issues/3347 + id 'com.google.cloud.tools.jib' version '3.4.0' apply(false) } group 'marginalia' @@ -29,11 +33,12 @@ subprojects.forEach {it -> reproducibleFileOrder = true } } - +ext { + dockerImageBase='container-registry.oracle.com/graalvm/jdk:21@sha256:1fd33d4d4eba3a9e1a41a728e39ea217178d257694eea1214fec68d2ed4d3d9b' +} allprojects { apply plugin: 'java' apply plugin: 'io.freefair.lombok' - dependencies { implementation libs.lombok testImplementation libs.lombok @@ -77,3 +82,4 @@ java { languageVersion.set(JavaLanguageVersion.of(21)) } } + diff --git a/code/api/assistant-api/readme.md b/code/api/assistant-api/readme.md deleted file mode 100644 index 7a35a592..00000000 --- a/code/api/assistant-api/readme.md +++ /dev/null @@ -1,8 +0,0 @@ -# Assistant API - -Client and models for talking to the [assistant-service](../../services-core/assistant-service), -implemented with the base client from [service-client](../../common/service-client). - -## Central Classes - -* [AssistantClient](src/main/java/nu/marginalia/assistant/client/AssistantClient.java) \ No newline at end of file diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java b/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java deleted file mode 100644 index a311aab1..00000000 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantClient.java +++ /dev/null @@ -1,159 +0,0 @@ -package nu.marginalia.assistant.client; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.assistant.api.AssistantApiGrpc; -import nu.marginalia.assistant.client.model.DictionaryResponse; -import nu.marginalia.assistant.client.model.DomainInformation; -import nu.marginalia.assistant.client.model.SimilarDomain; -import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.client.GrpcSingleNodeChannelPool; -import nu.marginalia.service.id.ServiceId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.*; - -import static nu.marginalia.assistant.client.AssistantProtobufCodec.*; - -@Singleton -public class AssistantClient { - private static final Logger logger = LoggerFactory.getLogger(AssistantClient.class); - - private final GrpcSingleNodeChannelPool channelPool; - private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); - @Inject - public AssistantClient(GrpcChannelPoolFactory factory) { - this.channelPool = factory.createSingle(ServiceId.Assistant, AssistantApiGrpc::newBlockingStub); - - } - - public Future dictionaryLookup(String word) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().dictionaryLookup( - DictionaryLookup.createRequest(word) - ); - - return DictionaryLookup.convertResponse(rsp); - }); - } - - @SuppressWarnings("unchecked") - public Future> spellCheck(String word) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().spellCheck( - SpellCheck.createRequest(word) - ); - - return SpellCheck.convertResponse(rsp); - }); - } - - public Map> spellCheck(List words, Duration timeout) throws InterruptedException { - List>>> tasks = new ArrayList<>(); - - for (String w : words) { - tasks.add(() -> { - var rsp = channelPool.api().spellCheck( - SpellCheck.createRequest(w) - ); - - return Map.entry(w, SpellCheck.convertResponse(rsp)); - }); - } - - var futures = virtualExecutorService.invokeAll(tasks, timeout.toMillis(), TimeUnit.MILLISECONDS); - Map> results = new HashMap<>(); - - for (var f : futures) { - if (!f.isDone()) - continue; - - var entry = f.resultNow(); - - results.put(entry.getKey(), entry.getValue()); - } - - return results; - } - - public Future unitConversion(String value, String from, String to) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().unitConversion( - UnitConversion.createRequest(from, to, value) - ); - - return UnitConversion.convertResponse(rsp); - }); - } - - public Future evalMath(String expression) { - return virtualExecutorService.submit(() -> { - var rsp = channelPool.api().evalMath( - EvalMath.createRequest(expression) - ); - - return EvalMath.convertResponse(rsp); - }); - } - - public Future> similarDomains(int domainId, int count) { - return virtualExecutorService.submit(() -> { - try { - var rsp = channelPool.api().getSimilarDomains( - DomainQueries.createRequest(domainId, count) - ); - - return DomainQueries.convertResponse(rsp); - } - catch (Exception e) { - logger.warn("Failed to get similar domains", e); - - throw e; - } - }); - } - - public Future> linkedDomains(int domainId, int count) { - return virtualExecutorService.submit(() -> { - try { - var rsp = channelPool.api().getLinkingDomains( - DomainQueries.createRequest(domainId, count) - ); - - return DomainQueries.convertResponse(rsp); - } - catch (Exception e) { - logger.warn("Failed to get linked domains", e); - throw e; - } - }); - - } - - public Future domainInformation(int domainId) { - return virtualExecutorService.submit(() -> { - try { - var rsp = channelPool.api().getDomainInfo( - DomainInfo.createRequest(domainId) - ); - - return DomainInfo.convertResponse(rsp); - } - catch (Exception e) { - logger.warn("Failed to get domain information", e); - - throw e; - } - }); - } - - public boolean isAccepting() { - return channelPool.hasChannel(); - } -} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java index 3c585263..225e0518 100644 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -13,7 +13,8 @@ import nu.marginalia.executor.upload.UploadDirContents; import nu.marginalia.executor.upload.UploadDirItem; import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import nu.marginalia.storage.model.FileStorageId; @@ -22,6 +23,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; +import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -39,180 +41,190 @@ public class ExecutorClient { { this.registry = registry; this.channelPool = grpcChannelPoolFactory - .createMulti(ServiceId.Executor, ExecutorApiGrpc::newBlockingStub); + .createMulti( + ServiceKey.forGrpcApi(ExecutorApiGrpc.class, ServicePartition.multi()), + ExecutorApiGrpc::newBlockingStub); } public void startFsm(int node, String actorName) { - channelPool.apiForNode(node).startFsm( - RpcFsmName.newBuilder() + channelPool.call(ExecutorApiBlockingStub::startFsm) + .forNode(node) + .run(RpcFsmName.newBuilder() .setActorName(actorName) - .build() - ); + .build()); + } public void stopFsm(int node, String actorName) { - channelPool.apiForNode(node).stopFsm( - RpcFsmName.newBuilder() + channelPool.call(ExecutorApiBlockingStub::stopFsm) + .forNode(node) + .run(RpcFsmName.newBuilder() .setActorName(actorName) - .build() - ); + .build()); } public void stopProcess(int node, String id) { - channelPool.apiForNode(node).stopProcess( - RpcProcessId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::stopProcess) + .forNode(node) + .run(RpcProcessId.newBuilder() .setProcessId(id) - .build() - ); + .build()); + } public void triggerCrawl(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerCrawl( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerCrawl) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void triggerRecrawl(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerRecrawl( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerRecrawl) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void triggerConvert(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerConvert( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerConvert) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void triggerConvertAndLoad(int node, FileStorageId fid) { - channelPool.apiForNode(node).triggerConvertAndLoad( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::triggerConvertAndLoad) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void loadProcessedData(int node, List ids) { - channelPool.apiForNode(node).loadProcessedData( - RpcFileStorageIds.newBuilder() + channelPool.call(ExecutorApiBlockingStub::loadProcessedData) + .forNode(node) + .run(RpcFileStorageIds.newBuilder() .addAllFileStorageIds(ids.stream().map(FileStorageId::id).toList()) - .build() - ); + .build()); } public void calculateAdjacencies(int node) { - channelPool.apiForNode(node).calculateAdjacencies(Empty.getDefaultInstance()); + channelPool.call(ExecutorApiBlockingStub::calculateAdjacencies) + .forNode(node) + .run(Empty.getDefaultInstance()); } public void sideloadEncyclopedia(int node, Path sourcePath, String baseUrl) { - channelPool.apiForNode(node).sideloadEncyclopedia( - RpcSideloadEncyclopedia.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadEncyclopedia) + .forNode(node) + .run(RpcSideloadEncyclopedia.newBuilder() .setBaseUrl(baseUrl) .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadDirtree(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadDirtree( - RpcSideloadDirtree.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadDirtree) + .forNode(node) + .run(RpcSideloadDirtree.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadReddit(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadReddit( - RpcSideloadReddit.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadReddit) + .forNode(node) + .run(RpcSideloadReddit.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadWarc(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadWarc( - RpcSideloadWarc.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadWarc) + .forNode(node) + .run(RpcSideloadWarc.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void sideloadStackexchange(int node, Path sourcePath) { - channelPool.apiForNode(node).sideloadStackexchange( - RpcSideloadStackexchange.newBuilder() + channelPool.call(ExecutorApiBlockingStub::sideloadStackexchange) + .forNode(node) + .run(RpcSideloadStackexchange.newBuilder() .setSourcePath(sourcePath.toString()) - .build() - ); + .build()); } public void createCrawlSpecFromDownload(int node, String description, String url) { - channelPool.apiForNode(node).createCrawlSpecFromDownload( - RpcCrawlSpecFromDownload.newBuilder() + channelPool.call(ExecutorApiBlockingStub::createCrawlSpecFromDownload) + .forNode(node) + .run(RpcCrawlSpecFromDownload.newBuilder() .setDescription(description) .setUrl(url) - .build() - ); + .build()); } public void exportAtags(int node, FileStorageId fid) { - channelPool.apiForNode(node).exportAtags( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportAtags) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void exportSampleData(int node, FileStorageId fid, int size, String name) { - channelPool.apiForNode(node).exportSampleData( - RpcExportSampleData.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportSampleData) + .forNode(node) + .run(RpcExportSampleData.newBuilder() .setFileStorageId(fid.id()) .setSize(size) .setName(name) - .build() - ); + .build()); } public void exportRssFeeds(int node, FileStorageId fid) { - channelPool.apiForNode(node).exportRssFeeds( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportRssFeeds) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void exportTermFrequencies(int node, FileStorageId fid) { - channelPool.apiForNode(node).exportTermFrequencies( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::exportTermFrequencies) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public void downloadSampleData(int node, String sampleSet) { - channelPool.apiForNode(node).downloadSampleData( - RpcDownloadSampleData.newBuilder() + channelPool.call(ExecutorApiBlockingStub::downloadSampleData) + .forNode(node) + .run(RpcDownloadSampleData.newBuilder() .setSampleSet(sampleSet) - .build() - ); + .build()); } public void exportData(int node) { - channelPool.apiForNode(node).exportData(Empty.getDefaultInstance()); + channelPool.call(ExecutorApiBlockingStub::exportData) + .forNode(node) + .run(Empty.getDefaultInstance()); } public void restoreBackup(int node, FileStorageId fid) { - channelPool.apiForNode(node).restoreBackup( - RpcFileStorageId.newBuilder() + channelPool.call(ExecutorApiBlockingStub::restoreBackup) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fid.id()) - .build() - ); + .build()); } public ActorRunStates getActorStates(int node) { try { - var rs = channelPool.apiForNode(node).getActorStates(Empty.getDefaultInstance()); + var rs = channelPool.call(ExecutorApiBlockingStub::getActorStates) + .forNode(node) + .run(Empty.getDefaultInstance()); var states = rs.getActorRunStatesList().stream() .map(r -> new ActorRunState( r.getActorName(), @@ -236,7 +248,9 @@ public class ExecutorClient { public UploadDirContents listSideloadDir(int node) { try { - var rs = channelPool.apiForNode(node).listSideloadDir(Empty.getDefaultInstance()); + var rs = channelPool.call(ExecutorApiBlockingStub::listSideloadDir) + .forNode(node) + .run(Empty.getDefaultInstance()); var items = rs.getEntriesList().stream() .map(i -> new UploadDirItem(i.getName(), i.getLastModifiedTime(), i.getIsDirectory(), i.getSize())) .toList(); @@ -252,11 +266,12 @@ public class ExecutorClient { public FileStorageContent listFileStorage(int node, FileStorageId fileId) { try { - var rs = channelPool.apiForNode(node).listFileStorage( - RpcFileStorageId.newBuilder() + var rs = channelPool.call(ExecutorApiBlockingStub::listFileStorage) + .forNode(node) + .run(RpcFileStorageId.newBuilder() .setFileStorageId(fileId.id()) .build() - ); + ); return new FileStorageContent(rs.getEntriesList().stream() .map(e -> new FileStorageFile(e.getName(), e.getSize(), e.getLastModifiedTime())) @@ -274,13 +289,13 @@ public class ExecutorClient { String uriPath = STR."/transfer/file/\{fileId.id()}"; String uriQuery = STR."path=\{URLEncoder.encode(path, StandardCharsets.UTF_8)}"; - var service = registry.getEndpoints(ApiSchema.REST, ServiceId.Executor, node) + var service = registry.getEndpoints(ServiceKey.forRest(ServiceId.Executor, node)) .stream().findFirst().orElseThrow(); try (var urlStream = service.endpoint().toURL(uriPath, uriQuery).openStream()) { urlStream.transferTo(destOutputStream); } - catch (IOException ex) { + catch (IOException | URISyntaxException ex) { throw new RuntimeException(ex); } } diff --git a/code/api/index-api/src/main/protobuf/index-api.proto b/code/api/index-api/src/main/protobuf/index-api.proto index ad05152e..521362f5 100644 --- a/code/api/index-api/src/main/protobuf/index-api.proto +++ b/code/api/index-api/src/main/protobuf/index-api.proto @@ -4,28 +4,6 @@ package actorapi; option java_package="nu.marginalia.index.api"; option java_multiple_files=true; -service IndexDomainLinksApi { - rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {} - rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {} - rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {} - rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {} - rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {} -} - -message RpcDomainId { - int32 domainId = 1; -} -message RpcDomainIdList { - repeated int32 domainId = 1 [packed=true]; -} -message RpcDomainIdCount { - int32 idCount = 1; -} -message RpcDomainIdPairs { - repeated int32 sourceIds = 1 [packed=true]; - repeated int32 destIds = 2 [packed=true]; -} - service QueryApi { rpc query(RpcQsQuery) returns (RpcQsResponse) {} } diff --git a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java index 74f78aa7..616b84c2 100644 --- a/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java +++ b/code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java @@ -9,14 +9,12 @@ import nu.marginalia.query.model.QueryParams; import nu.marginalia.query.model.QueryResponse; import nu.marginalia.service.client.GrpcChannelPoolFactory; import nu.marginalia.service.client.GrpcSingleNodeChannelPool; -import nu.marginalia.service.id.ServiceId; -import org.roaringbitmap.longlong.PeekableLongIterator; -import org.roaringbitmap.longlong.Roaring64Bitmap; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.CheckReturnValue; -import java.util.List; @Singleton public class QueryClient { @@ -27,134 +25,25 @@ public class QueryClient { .register(); private final GrpcSingleNodeChannelPool queryApiPool; - private final GrpcSingleNodeChannelPool domainLinkApiPool; private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject public QueryClient(GrpcChannelPoolFactory channelPoolFactory) { - this.queryApiPool = channelPoolFactory.createSingle(ServiceId.Query, QueryApiGrpc::newBlockingStub); - this.domainLinkApiPool = channelPoolFactory.createSingle(ServiceId.Query, IndexDomainLinksApiGrpc::newBlockingStub); + this.queryApiPool = channelPoolFactory.createSingle( + ServiceKey.forGrpcApi(QueryApiGrpc.class, ServicePartition.any()), + QueryApiGrpc::newBlockingStub); } @CheckReturnValue public QueryResponse search(QueryParams params) { var query = QueryProtobufCodec.convertQueryParams(params); - return wmsa_qs_api_search_time.time( - () -> QueryProtobufCodec.convertQueryResponse(queryApiPool - .importantCall((api) -> api.query(query)) + return wmsa_qs_api_search_time.time(() -> + QueryProtobufCodec.convertQueryResponse( + queryApiPool.call(QueryApiGrpc.QueryApiBlockingStub::query).run(query) ) ); } - public AllLinks getAllDomainLinks() { - AllLinks links = new AllLinks(); - - domainLinkApiPool.api() - .getAllLinks(Empty.getDefaultInstance()) - .forEachRemaining(pairs -> { - for (int i = 0; i < pairs.getDestIdsCount(); i++) { - links.add(pairs.getSourceIds(i), pairs.getDestIds(i)); - } - }); - - return links; - } - - public List getLinksToDomain(int domainId) { - try { - return domainLinkApiPool.api() - .getLinksToDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getDomainIdList() - .stream() - .sorted() - .toList(); - } - catch (Exception e) { - logger.error("API Exception", e); - return List.of(); - } - } - - public List getLinksFromDomain(int domainId) { - try { - return domainLinkApiPool.api() - .getLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getDomainIdList() - .stream() - .sorted() - .toList(); - } - catch (Exception e) { - logger.error("API Exception", e); - return List.of(); - } - } - - public int countLinksToDomain(int domainId) { - try { - return domainLinkApiPool.api() - .countLinksToDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getIdCount(); - } - catch (Exception e) { - logger.error("API Exception", e); - return 0; - } - } - - public int countLinksFromDomain(int domainId) { - try { - return domainLinkApiPool.api() - .countLinksFromDomain(RpcDomainId - .newBuilder() - .setDomainId(domainId) - .build()) - .getIdCount(); - } - catch (Exception e) { - logger.error("API Exception", e); - return 0; - } - } - public static class AllLinks { - private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap(); - - public void add(int source, int dest) { - sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest)); - } - - public Iterator iterator() { - return new Iterator(); - } - - public class Iterator { - private final PeekableLongIterator base = sourceToDest.getLongIterator(); - long val = Long.MIN_VALUE; - - public boolean advance() { - if (base.hasNext()) { - val = base.next(); - return true; - } - return false; - } - public int source() { - return (int) (val >>> 32); - } - public int dest() { - return (int) (val & 0xFFFF_FFFFL); - } - } - } } diff --git a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java index e65c5529..b520f6e2 100644 --- a/code/common/config/src/main/java/nu/marginalia/WmsaHome.java +++ b/code/common/config/src/main/java/nu/marginalia/WmsaHome.java @@ -5,8 +5,7 @@ import nu.marginalia.service.ServiceHomeNotConfiguredException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Optional; +import java.util.Objects; import java.util.stream.Stream; public class WmsaHome { @@ -26,15 +25,28 @@ public class WmsaHome { } public static Path getHomePath() { - var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElseGet(WmsaHome::findDefaultHomePath); + String[] possibleLocations = new String[] { + System.getenv("WMSA_HOME"), + System.getProperty("system.homePath"), + "/var/lib/wmsa", + "/wmsa" + }; + + String retStr = Stream.of(possibleLocations) + .filter(Objects::nonNull) + .map(Path::of) + .filter(Files::isDirectory) + .map(Path::toString) + .findFirst() + .orElseThrow(() -> + new ServiceHomeNotConfiguredException(""" + Could not find $WMSA_HOME, either set environment + variable, the 'system.homePath' property, + or ensure either /wmssa or /var/lib/wmsa exists + """)); var ret = Path.of(retStr); - if (!Files.isDirectory(ret)) { - throw new ServiceHomeNotConfiguredException("Could not find $WMSA_HOME, either set environment variable or ensure " + retStr + " exists"); - } - - if (!Files.isDirectory(ret.resolve("model"))) { throw new ServiceHomeNotConfiguredException("You need to run 'run/setup.sh' to download models to run/ before this will work!"); } @@ -42,22 +54,6 @@ public class WmsaHome { return ret; } - private static String findDefaultHomePath() { - - // Assume this is a local developer and not a production system, since it would have WMSA_HOME set. - // Developers probably have a "run/" somewhere upstream from cwd. - // - - return Stream.iterate(Paths.get("").toAbsolutePath(), f -> f != null && Files.exists(f), Path::getParent) - .filter(p -> Files.exists(p.resolve("run/env"))) - .filter(p -> Files.exists(p.resolve("run/setup.sh"))) - .map(p -> p.resolve("run")) - .findAny() - .orElse(Path.of("/var/lib/wmsa")) - .toString(); - } - - public static Path getAdsDefinition() { return getHomePath().resolve("data").resolve("adblock.txt"); } diff --git a/code/common/service-discovery/build.gradle b/code/common/service-discovery/build.gradle index c274b66a..d86ef9f4 100644 --- a/code/common/service-discovery/build.gradle +++ b/code/common/service-discovery/build.gradle @@ -6,6 +6,11 @@ plugins { repositories { mavenLocal() mavenCentral() + + repositories { + mavenCentral() + maven { url 'https://jitpack.io' } + } } java { @@ -18,6 +23,7 @@ dependencies { implementation libs.bundles.curator implementation libs.guice + implementation libs.bundles.gson implementation libs.bundles.mariadb implementation libs.bundles.grpc implementation libs.notnull @@ -29,4 +35,5 @@ dependencies { testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') testImplementation 'org.testcontainers:mariadb:1.17.4' testImplementation 'org.testcontainers:junit-jupiter:1.17.4' + testImplementation project(':code:functions:math:api') } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java index 161c2a9e..cc783c72 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceDiscoveryModule.java @@ -1,7 +1,6 @@ package nu.marginalia.service; import com.google.inject.AbstractModule; -import nu.marginalia.service.discovery.FixedServiceRegistry; import nu.marginalia.service.discovery.ServiceRegistryIf; import nu.marginalia.service.discovery.ZkServiceRegistry; import org.apache.curator.framework.CuratorFramework; @@ -18,18 +17,13 @@ public class ServiceDiscoveryModule extends AbstractModule { private static final Logger logger = LoggerFactory.getLogger(ServiceDiscoveryModule.class); public void configure() { - getZookeeperHosts().ifPresentOrElse((hosts) -> { - logger.info("Using Zookeeper service registry at {}", hosts); - CuratorFramework client = CuratorFrameworkFactory - .newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000)); + var hosts = getZookeeperHosts().orElseThrow(() -> new IllegalStateException("Zookeeper hosts not set")); + logger.info("Using Zookeeper service registry at {}", hosts); + CuratorFramework client = CuratorFrameworkFactory + .newClient(hosts, new ExponentialBackoffRetry(100, 10, 1000)); - bind(CuratorFramework.class).toInstance(client); - bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); - }, - () -> { - logger.info("Using fixed service registry"); - bind(ServiceRegistryIf.class).to(FixedServiceRegistry.class); - }); + bind(CuratorFramework.class).toInstance(client); + bind(ServiceRegistryIf.class).to(ZkServiceRegistry.class); } private Optional getZookeeperHosts() { diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java index 181428e1..504abdc0 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/ServiceHomeNotConfiguredException.java @@ -1,10 +1,6 @@ package nu.marginalia.service; public class ServiceHomeNotConfiguredException extends RuntimeException { - - public ServiceHomeNotConfiguredException() { - super("WMSA_HOME environment variable not set"); - } public ServiceHomeNotConfiguredException(String message) { super(message); } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java index 087fc666..a4d0035a 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcChannelPoolFactory.java @@ -6,8 +6,10 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import nu.marginalia.service.NodeConfigurationWatcher; import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import java.util.function.Function; @@ -26,30 +28,32 @@ public class GrpcChannelPoolFactory { } /** Create a new multi-node channel pool for the given service. */ - public GrpcMultiNodeChannelPool createMulti(ServiceId serviceId, + public GrpcMultiNodeChannelPool createMulti(ServiceKey key, Function stubConstructor) { return new GrpcMultiNodeChannelPool<>(serviceRegistryIf, - serviceId, + key, this::createChannel, stubConstructor, nodeConfigurationWatcher); } /** Create a new single-node channel pool for the given service. */ - public GrpcSingleNodeChannelPool createSingle(ServiceId serviceId, + public GrpcSingleNodeChannelPool createSingle(ServiceKey key, Function stubConstructor) { - return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, serviceId, - new NodeSelectionStrategy.Any(), - this::createChannel, - stubConstructor); + return new GrpcSingleNodeChannelPool<>(serviceRegistryIf, key, this::createChannel, stubConstructor); } - private ManagedChannel createChannel(InstanceAddress route) { - return ManagedChannelBuilder + private ManagedChannel createChannel(InstanceAddress route) { + + var mc = ManagedChannelBuilder .forAddress(route.host(), route.port()) .usePlaintext() .build(); + + mc.getState(true); + + return mc; } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java index f3916338..de76ecc9 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcMultiNodeChannelPool.java @@ -4,13 +4,17 @@ import io.grpc.ManagedChannel; import lombok.SneakyThrows; import nu.marginalia.service.NodeConfigurationWatcher; import nu.marginalia.service.discovery.ServiceRegistryIf; +import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Stream; @@ -22,21 +26,20 @@ public class GrpcMultiNodeChannelPool { private final ConcurrentHashMap> pools = new ConcurrentHashMap<>(); private static final Logger logger = LoggerFactory.getLogger(GrpcMultiNodeChannelPool.class); - private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); private final ServiceRegistryIf serviceRegistryIf; - private final ServiceId serviceId; - private final Function, ManagedChannel> channelConstructor; + private final ServiceKey serviceKey; + private final Function channelConstructor; private final Function stubConstructor; private final NodeConfigurationWatcher nodeConfigurationWatcher; @SneakyThrows public GrpcMultiNodeChannelPool(ServiceRegistryIf serviceRegistryIf, - ServiceId serviceId, - Function, ManagedChannel> channelConstructor, + ServiceKey serviceKey, + Function channelConstructor, Function stubConstructor, NodeConfigurationWatcher nodeConfigurationWatcher) { this.serviceRegistryIf = serviceRegistryIf; - this.serviceId = serviceId; + this.serviceKey = serviceKey; this.channelConstructor = channelConstructor; this.stubConstructor = stubConstructor; this.nodeConfigurationWatcher = nodeConfigurationWatcher; @@ -51,51 +54,74 @@ public class GrpcMultiNodeChannelPool { return pools.computeIfAbsent(node, _ -> new GrpcSingleNodeChannelPool<>( serviceRegistryIf, - serviceId, - new NodeSelectionStrategy.Just(node), + serviceKey.forPartition(ServicePartition.partition(node)), channelConstructor, stubConstructor)); } - - /** Get an API stub for the given node */ - public STUB apiForNode(int node) { - return pools.computeIfAbsent(node, this::getPoolForNode).api(); - } - - - /** Invoke a function on each node, returning a list of futures in a terminal state, as per - * ExecutorService$invokeAll */ - public List> invokeAll(Function> callF) throws InterruptedException { - List> calls = getEligibleNodes().stream() - .mapMulti(this::passNodeIfOk) - .map(callF) - .toList(); - - return virtualExecutorService.invokeAll(calls); - } - - /** Invoke a function on each node, returning a stream of results */ - public Stream callEachSequential(Function call) { - return getEligibleNodes().stream() - .mapMulti(this::passNodeIfOk) - .map(call); - } - - // Eat connectivity exceptions and log them when doing a broadcast-style calls - private void passNodeIfOk(Integer nodeId, Consumer consumer) { - try { - consumer.accept(apiForNode(nodeId)); - } - catch (Exception ex) { - logger.error("Error calling node {}", nodeId, ex); - } - } - /** Get the list of nodes that are eligible for broadcast-style requests */ public List getEligibleNodes() { return nodeConfigurationWatcher.getQueryNodes(); } + public CallBuilderBase call(BiFunction method) { + return new CallBuilderBase<>(method); + } + + public class CallBuilderBase { + private final BiFunction method; + + private CallBuilderBase(BiFunction method) { + this.method = method; + } + + public GrpcSingleNodeChannelPool.CallBuilderBase forNode(int node) { + return getPoolForNode(node).call(method); + } + + public List run(I arg) { + return getEligibleNodes().stream() + .map(node -> getPoolForNode(node).call(method).run(arg)) + .toList(); + } + + public CallBuilderAsync async(ExecutorService service) { + return new CallBuilderAsync<>(service, method); + } + } + + public class CallBuilderAsync { + private final Executor executor; + private final BiFunction method; + + public CallBuilderAsync(Executor executor, BiFunction method) { + this.executor = executor; + this.method = method; + } + + public CompletableFuture> runAll(I arg) { + var futures = getEligibleNodes().stream() + .map(GrpcMultiNodeChannelPool.this::getPoolForNode) + .map(pool -> + pool.call(method) + .async(executor) + .run(arg) + ).toList(); + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply(v -> futures.stream().map(CompletableFuture::join).toList()); + } + + public List> runEach(I arg) { + return getEligibleNodes().stream() + .map(GrpcMultiNodeChannelPool.this::getPoolForNode) + .map(pool -> + pool.call(method) + .async(executor) + .run(arg) + ).toList(); + + } + } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java index 137e2933..68c33277 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java @@ -1,142 +1,232 @@ package nu.marginalia.service.client; import com.google.common.collect.Sets; -import io.grpc.ConnectivityState; import io.grpc.ManagedChannel; import lombok.SneakyThrows; import nu.marginalia.service.discovery.ServiceRegistryIf; import nu.marginalia.service.discovery.monitor.ServiceChangeMonitor; -import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.service.discovery.property.PartitionTraits; import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Function; /** A pool of gRPC channels for a service, with a separate channel for each node. *

* Manages unicast-style requests */ public class GrpcSingleNodeChannelPool extends ServiceChangeMonitor { - private final Map, ManagedChannel> channels = new ConcurrentHashMap<>(); - private final Map>> routes = new ConcurrentHashMap<>(); + private final Map channels = new ConcurrentHashMap<>(); private static final Logger logger = LoggerFactory.getLogger(GrpcSingleNodeChannelPool.class); private final ServiceRegistryIf serviceRegistryIf; - private final ServiceId serviceId; - private final NodeSelectionStrategy nodeSelectionStrategy; - private final Function, ManagedChannel> channelConstructor; + private final Function channelConstructor; private final Function stubConstructor; + @SneakyThrows public GrpcSingleNodeChannelPool(ServiceRegistryIf serviceRegistryIf, - ServiceId serviceId, - NodeSelectionStrategy nodeSelectionStrategy, - Function, ManagedChannel> channelConstructor, + ServiceKey serviceKey, + Function channelConstructor, Function stubConstructor) { - super(serviceId); + super(serviceKey); this.serviceRegistryIf = serviceRegistryIf; - this.serviceId = serviceId; - this.nodeSelectionStrategy = nodeSelectionStrategy; this.channelConstructor = channelConstructor; this.stubConstructor = stubConstructor; serviceRegistryIf.registerMonitor(this); onChange(); + + awaitChannel(Duration.ofSeconds(5)); } @Override - public boolean onChange() { - switch (nodeSelectionStrategy) { - case NodeSelectionStrategy.Any() -> - serviceRegistryIf - .getServiceNodes(serviceId) - .forEach(this::refreshNode); - case NodeSelectionStrategy.Just(int node) -> - refreshNode(node); + public synchronized boolean onChange() { + Set newRoutes = serviceRegistryIf.getEndpoints(serviceKey); + Set oldRoutes = new HashSet<>(channels.keySet()); + + // Find the routes that have been added or removed + for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) { + ConnectionHolder oldChannel; + if (newRoutes.contains(route)) { + logger.info("Adding route {}", route); + oldChannel = channels.put(route, new ConnectionHolder(route)); + } else { + logger.info("Expelling route {}", route); + oldChannel = channels.remove(route); + } + if (oldChannel != null) { + oldChannel.close(); + } } return true; } - private void refreshNode(int node) { + private class ConnectionHolder implements Comparable { + private final AtomicReference channel = new AtomicReference<>(); + private final InstanceAddress address; - Set> newRoutes = serviceRegistryIf.getEndpoints(ApiSchema.GRPC, serviceId, node); - Set> oldRoutes = routes.getOrDefault(node, Set.of()); - - // Find the routes that have been added or removed - for (var route : Sets.symmetricDifference(oldRoutes, newRoutes)) { - - ManagedChannel oldChannel; - - if (newRoutes.contains(route)) { - var newChannel = channelConstructor.apply(route); - oldChannel = channels.put(route, newChannel); - } else { - oldChannel = channels.remove(route); - } - - if (oldChannel != null) - oldChannel.shutdown(); + ConnectionHolder(InstanceAddress address) { + this.address = address; } - routes.put(node, newRoutes); + public ManagedChannel get() { + var value = channel.get(); + if (value != null) { + return value; + } + + try { + logger.info("Creating channel for {}:{}", serviceKey, address); + value = channelConstructor.apply(address); + if (channel.compareAndSet(null, value)) { + return value; + } + else { + value.shutdown(); + return channel.get(); + } + } + catch (Exception e) { + logger.error(STR."Failed to get channel for \{address}", e); + return null; + } + } + + public void close() { + ManagedChannel mc = channel.getAndSet(null); + if (mc != null) { + mc.shutdown(); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConnectionHolder that = (ConnectionHolder) o; + return Objects.equals(address, that.address); + } + + + @Override + public int hashCode() { + return Objects.hash(address); + } + + @Override + public int compareTo(@NotNull GrpcSingleNodeChannelPool.ConnectionHolder o) { + + return -Long.compare(address.cxTime(), o.address.cxTime()); // Reverse order + } } + + public boolean hasChannel() { return !channels.isEmpty(); } - /** Get an API stub for the given node */ - public STUB api() { - return stubConstructor.apply(getChannel()); + public synchronized boolean awaitChannel(Duration timeout) throws InterruptedException { + if (hasChannel()) return true; + + final long endTime = System.currentTimeMillis() + timeout.toMillis(); + + while (!hasChannel()) { + long timeLeft = endTime - System.currentTimeMillis(); + if (timeLeft <= 0) return false; + this.wait(timeLeft); + } + return hasChannel(); } - /** Try to make the call go through. The function will cycle through - * available routes until exhaustion, and only then will it give up - */ - public T importantCall(Function function) { - for (int i = 0; i < channels.size(); i++) { + private T call(BiFunction call, I arg) throws RuntimeException { + final List exceptions = new ArrayList<>(); + final List connectionHolders = new ArrayList<>(channels.values()); + + // Randomize the order of the connection holders to spread out the load + Collections.shuffle(connectionHolders); + + for (var channel : connectionHolders) { try { - return function.apply(api()); + return call.apply(stubConstructor.apply(channel.get()), arg); } catch (Exception e) { - logger.error("API Exception", e); + exceptions.add(e); } } - throw new ServiceNotAvailableException(serviceId); + for (var e : exceptions) { + logger.error("Failed to call service {}", serviceKey, e); + } + + throw new ServiceNotAvailableException(serviceKey); } - /** Get the channel that is most ready to use */ - public ManagedChannel getChannel() { - return channels - .values() - .stream() - .min(this::compareChannelsByState) - .orElseThrow(() -> new ServiceNotAvailableException(serviceId)); + public CallBuilderBase call(BiFunction method) { + return new CallBuilderBase<>(method); } - /** Sort the channels by how ready they are to use */ - private int compareChannelsByState(ManagedChannel a, ManagedChannel b) { - var aState = a.getState(true); - var bState = b.getState(true); + public class CallBuilderBase { + private final BiFunction method; + private CallBuilderBase(BiFunction method) { + this.method = method; + } - if (aState == ConnectivityState.READY) return -1; - if (bState == ConnectivityState.READY) return 1; - if (aState == ConnectivityState.CONNECTING) return -1; - if (bState == ConnectivityState.CONNECTING) return 1; - if (aState == ConnectivityState.IDLE) return -1; - if (bState == ConnectivityState.IDLE) return 1; + public T run(I arg) { + return call(method, arg); + } - return 0; + public List runFor(I... args) { + return runFor(List.of(args)); + } + + public List runFor(List args) { + List results = new ArrayList<>(); + for (var arg : args) { + results.add(call(method, arg)); + } + return results; + } + public CallBuilderAsync async(Executor executor) { + return new CallBuilderAsync<>(executor, method); + } } + public class CallBuilderAsync { + private final Executor executor; + private final BiFunction method; + public CallBuilderAsync(Executor executor, BiFunction method) { + this.executor = executor; + this.method = method; + } + public CompletableFuture run(I arg) { + return CompletableFuture.supplyAsync(() -> call(method, arg), executor); + } + public CompletableFuture> runFor(List args) { + List> results = new ArrayList<>(); + for (var arg : args) { + results.add(CompletableFuture.supplyAsync(() -> call(method, arg), executor)); + } + return CompletableFuture.allOf(results.toArray(new CompletableFuture[0])) + .thenApply(v -> results.stream().map(CompletableFuture::join).toList()); + } + public CompletableFuture> runFor(I... args) { + return runFor(List.of(args)); + } + } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java deleted file mode 100644 index 7e040597..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/NodeSelectionStrategy.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.service.client; - -public sealed interface NodeSelectionStrategy { - boolean test(int node); - record Any() implements NodeSelectionStrategy { - @Override - public boolean test(int node) { - return true; - } - } - record Just(int node) implements NodeSelectionStrategy { - @Override - public boolean test(int node) { - return this.node == node; - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java index e11287ab..b905beab 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/client/ServiceNotAvailableException.java @@ -1,12 +1,9 @@ package nu.marginalia.service.client; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; public class ServiceNotAvailableException extends RuntimeException { - public ServiceNotAvailableException(ServiceId id, int node) { - super(STR."Service \{id} not available on node \{node}"); - } - public ServiceNotAvailableException(ServiceId id) { - super(STR."Service \{id} not available"); + public ServiceNotAvailableException(ServiceKey key) { + super(STR."Service \{key} not available"); } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java deleted file mode 100644 index e8254cfd..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/FixedServiceRegistry.java +++ /dev/null @@ -1,122 +0,0 @@ -package nu.marginalia.service.discovery; - -import com.google.inject.Inject; -import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.service.discovery.monitor.*; -import nu.marginalia.service.discovery.property.ApiSchema; -import nu.marginalia.service.discovery.property.ServiceEndpoint; -import nu.marginalia.service.discovery.property.ServiceEndpoint.GrpcEndpoint; -import nu.marginalia.service.discovery.property.ServiceEndpoint.InstanceAddress; -import nu.marginalia.service.discovery.property.ServiceEndpoint.RestEndpoint; -import nu.marginalia.service.id.ServiceId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.SQLException; -import java.time.Duration; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; - -/** A service registry that returns fixed endpoints for all services. - *

- * This is for backwards-compatibility with old docker-compose files with no - * ZooKeeper configured. - * */ -public class FixedServiceRegistry implements ServiceRegistryIf { - private static final Logger logger = LoggerFactory.getLogger(FixedServiceRegistry.class); - - private final HikariDataSource dataSource; - - @Inject - public FixedServiceRegistry(HikariDataSource dataSource) { - this.dataSource = dataSource; - } - - @Override - public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, int node, UUID instanceUUID, String externalAddress) throws Exception { - return switch (schema) { - case REST -> new ServiceEndpoint.RestEndpoint(externalAddress, 80); - case GRPC -> new ServiceEndpoint.GrpcEndpoint(externalAddress, 81); - }; - } - - @Override - public void announceInstance(ServiceId id, int node, UUID instanceUUID) { - // No-op - } - - @Override - public Set getServiceNodes(ServiceId id) { - - if (id == ServiceId.Executor || id == ServiceId.Index) { - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement("SELECT ID FROM NODE_CONFIGURATION")) { - Set ret = new HashSet<>(); - var rs = stmt.executeQuery(); - while (rs.next()) { - ret.add(rs.getInt(1)); - } - return ret; - } - catch (SQLException ex) { - return Set.of(); - } - } - - else return Set.of(0); - } - - @Override - public int requestPort(String externalHost, ApiSchema schema, ServiceId id, int node) { - return switch(schema) { - case REST -> 80; - case GRPC -> 81; - }; - } - - @Override - public Set> getEndpoints(ApiSchema schema, ServiceId id, int node) { - return switch (schema) { - case REST -> Set.of(new InstanceAddress<>( - new RestEndpoint(id.serviceName + "-" + node, 80), - UUID.randomUUID())); - case GRPC -> Set.of(new InstanceAddress<>( - new GrpcEndpoint(id.serviceName + "-" + node, 81), - UUID.randomUUID())); - }; - } - - public void registerMonitor(ServiceMonitorIf monitor) throws Exception { - // We don't have any notification mechanism, so we just periodically - // invoke the monitor's onChange method to simulate it. - - periodicallyInvoke(monitor, Duration.ofSeconds(15)); - } - - - void periodicallyInvoke(ServiceMonitorIf monitor, Duration d) { - Thread.ofPlatform().name("PeriodicInvoker").start(() -> { - for (;;) { - try { - Thread.sleep(d); - } catch (InterruptedException e) { - break; - } - - boolean reRegister; - try { - reRegister = monitor.onChange(); - } - catch (Exception ex) { - logger.error("Monitor failed", ex); - reRegister = true; - } - - if (!reRegister) { - break; - } - } - }); - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java index b3460418..04877bd3 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ServiceRegistryIf.java @@ -1,10 +1,10 @@ package nu.marginalia.service.discovery; import nu.marginalia.service.discovery.monitor.*; -import nu.marginalia.service.discovery.property.ApiSchema; import nu.marginalia.service.discovery.property.ServiceEndpoint; import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; -import nu.marginalia.service.id.ServiceId; + +import nu.marginalia.service.discovery.property.ServiceKey; import java.util.Set; import java.util.UUID; @@ -16,40 +16,33 @@ public interface ServiceRegistryIf { /** * Register a service with the registry. *

- * Once the instance has announced itself with {@link #announceInstance(ServiceId id, int node, UUID instanceUUID) announceInstance(...)}, - * the service will be available for discovery with {@link #getEndpoints(ApiSchema schema, ServiceId id, int node) getEndpoints(...)}. + * Once the instance has announced itself with {@link #announceInstance(UUID instanceUUID) announceInstance(...)}, + * the service will be available for discovery with {@link #getEndpoints(ServiceKey key) getEndpoints(...)}. * - * @param schema the API schema - * @param id the service identifier - * @param node the node number + * @param key the key identifying the service * @param instanceUUID the unique UUID of the instance * @param externalAddress the public address of the service */ - ServiceEndpoint registerService(ApiSchema schema, - ServiceId id, - int node, + ServiceEndpoint registerService(ServiceKey key, UUID instanceUUID, String externalAddress) throws Exception; + + void declareFirstBoot(); + void waitForFirstBoot() throws InterruptedException; + /** Let the world know that the service is running * and ready to accept requests. */ - void announceInstance(ServiceId id, int node, UUID instanceUUID); - - /** Return all nodes that are running for the specified service. */ - Set getServiceNodes(ServiceId id); + void announceInstance(UUID instanceUUID); /** At the discretion of the implementation, provide a port that is unique - * across (externalHost, serviceId, schema, node). It may be randomly selected + * across (host, api-schema). It may be randomly selected * or hard-coded or some combination of behaviors. */ - int requestPort(String externalHost, - ApiSchema schema, - ServiceId id, - int node); + int requestPort(String externalHost, ServiceKey key); /** Get all endpoints for the service on the specified node and schema. */ - Set> - getEndpoints(ApiSchema schema, ServiceId id, int node); + Set getEndpoints(ServiceKey schema); /** Register a monitor to be notified when the service registry changes. *

@@ -61,9 +54,6 @@ public interface ServiceRegistryIf { * monitor type. *
    *
  • {@link ServiceChangeMonitor} is notified when any node for the service changes.
  • - *
  • {@link ServiceNodeChangeMonitor} is notified when a specific node for the service changes.
  • - *
  • {@link ServiceRestEndpointChangeMonitor} is notified when the REST endpoints for the specified node service changes.
  • - *
  • {@link ServiceGrpcEndpointChangeMonitor} is notified when the gRPC endpoints for the specified node service changes.
  • *
* */ void registerMonitor(ServiceMonitorIf monitor) throws Exception; diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java index 48efc64f..1af51d75 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/ZkServiceRegistry.java @@ -4,10 +4,10 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; import nu.marginalia.service.discovery.monitor.*; -import nu.marginalia.service.discovery.property.ApiSchema; import nu.marginalia.service.discovery.property.ServiceEndpoint; import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; -import nu.marginalia.service.id.ServiceId; + +import nu.marginalia.service.discovery.property.ServiceKey; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorWatcher; import org.apache.curator.utils.ZKPaths; @@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; /** A versatile service registry that uses ZooKeeper to store service endpoints. * It is used to register services and to look up the endpoints of other services. @@ -35,6 +34,8 @@ public class ZkServiceRegistry implements ServiceRegistryIf { private static final Logger logger = LoggerFactory.getLogger(ZkServiceRegistry.class); private volatile boolean stopped = false; + private final List livenessPaths = new ArrayList<>(); + @Inject @SneakyThrows public ZkServiceRegistry(CuratorFramework curatorFramework) { @@ -51,93 +52,98 @@ public class ZkServiceRegistry implements ServiceRegistryIf { } @Override - public ServiceEndpoint registerService(ApiSchema schema, ServiceId id, - int node, + public ServiceEndpoint registerService(ServiceKey key, UUID instanceUUID, String externalAddress) throws Exception { - var ephemeralProperty = curatorFramework.create() - .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL); + var endpoint = new ServiceEndpoint(externalAddress, requestPort(externalAddress, key)); - var endpoint = ServiceEndpoint.forSchema(schema, externalAddress, - requestPort(externalAddress, schema, id, node) - ); - - String path; - byte[] payload; - - switch (endpoint) { - case ServiceEndpoint.GrpcEndpoint(String host, int port) -> { - path = STR."/services/\{id.serviceName}/\{node}/grpc/\{instanceUUID.toString()}"; - payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8); - } - case ServiceEndpoint.RestEndpoint(String host, int port) -> { - path = STR."/services/\{id.serviceName}/\{node}/rest/\{instanceUUID.toString()}"; - payload = STR."\{host}:\{port}".getBytes(StandardCharsets.UTF_8); - } - } + String path = STR."\{key.toPath()}/\{instanceUUID.toString()}"; + byte[] payload = STR."\{endpoint.host()}:\{endpoint.port()}".getBytes(StandardCharsets.UTF_8); logger.info("Registering {} -> {}", path, endpoint); - ephemeralProperty.forPath(path, payload); + curatorFramework.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL) + .forPath(path, payload); return endpoint; } + @SneakyThrows @Override - public void announceInstance(ServiceId id, int node, UUID instanceUUID) { + public void declareFirstBoot() { + if (!isFirstBoot()) { + curatorFramework.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(STR."/first-boot"); + } + } + + @Override + public void waitForFirstBoot() throws InterruptedException { + if (!isFirstBoot()) + logger.info("Waiting for first-boot flag"); + + while (true) { + if (isFirstBoot()) + return; + + Thread.sleep(1000); + } + } + + private boolean isFirstBoot() { try { - String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}"; + return curatorFramework.checkExists().forPath("/first-boot") != null; + } + catch (Exception ex) { + logger.error("Failed to check first-boot", ex); + return false; + } + } + + @Override + public void announceInstance(UUID instanceUUID) { + try { + String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}"; + + livenessPaths.add(serviceRoot); + curatorFramework.create() .creatingParentsIfNeeded() .withMode(CreateMode.EPHEMERAL) .forPath(serviceRoot); } catch (Exception ex) { - logger.error("Failed to create service root for {}", id.serviceName); + logger.error("Failed to create service root for {}", instanceUUID); } } /** * Returns true if the service has announced itself as up and running. */ - public boolean isInstanceRunning(ServiceId id, int node, UUID instanceUUID) { + public boolean isInstanceRunning(UUID instanceUUID) { try { - String serviceRoot = STR."/services/\{id.serviceName}/\{node}/running/\{instanceUUID.toString()}"; + String serviceRoot = STR."/running-instances/\{instanceUUID.toString()}"; return null != curatorFramework.checkExists().forPath(serviceRoot); } catch (Exception ex) { - logger.error("Failed to check if service is running {}", id.serviceName); + logger.error("Failed to check if instance is running {}", instanceUUID); return false; } } - @Override - public Set getServiceNodes(ServiceId id) { - try { - String serviceRoot = STR."/services/\{id.serviceName}"; - return curatorFramework.getChildren().forPath(serviceRoot) - .stream().map(Integer::parseInt) - .collect(Collectors.toSet()); - } - catch (Exception ex) { - logger.error("Failed to get nodes for service {}", id.serviceName); - return Set.of(); - } - } - @Override public int requestPort(String externalHost, - ApiSchema schema, - ServiceId id, - int node) - { + ServiceKey key) { if (!Boolean.getBoolean("service.random-port")) { - return switch(schema) { - case REST -> 80; - case GRPC -> 81; + return switch (key) { + case ServiceKey.Rest rest -> 80; + case ServiceKey.Grpc grpc -> 81; }; } @@ -146,9 +152,9 @@ public class ZkServiceRegistry implements ServiceRegistryIf { var random = new Random(); - String host = STR."\{id.serviceName}-\{node}"; + String identifier = key.toPath(); - byte[] payload = STR."\{schema}://\{host}".getBytes(StandardCharsets.UTF_8); + byte[] payload = identifier.getBytes(); for (int iter = 0; iter < 1000; iter++) { try { @@ -161,7 +167,7 @@ public class ZkServiceRegistry implements ServiceRegistryIf { return port; } catch (Exception ex) { - logger.error(STR."Still negotiating port for \{schema}://\{id.serviceName}:\{node}"); + logger.error(STR."Still negotiating port for \{identifier}"); } } @@ -169,81 +175,30 @@ public class ZkServiceRegistry implements ServiceRegistryIf { } @Override - public Set> getEndpoints(ApiSchema schema, ServiceId id, int node) { - return switch (schema) { - case REST -> getRestEndpoints(id, node); - case GRPC -> getGrpcEndpoints(id, node); - }; - } - - public Set> getRestEndpoints(ServiceId id, int node) { + public Set getEndpoints(ServiceKey key) { try { - Set> ret = new HashSet<>(); - String restRoot = STR."/services/\{id.serviceName}/\{node}/rest"; + Set ret = new HashSet<>(); for (var uuid : curatorFramework .getChildren() - .forPath(restRoot)) { + .forPath(key.toPath())) { - if (!isInstanceRunning(id, node, UUID.fromString(uuid))) { + if (!isInstanceRunning(UUID.fromString(uuid))) { continue; } + var path = ZKPaths.makePath(key.toPath(), uuid); byte[] data = curatorFramework .getData() - .forPath(ZKPaths.makePath(restRoot, uuid)); - String hostAndPort = new String(data); - var address = RestEndpoint - .parse(hostAndPort) - .asInstance(UUID.fromString(uuid)); + .forPath(path); - // Ensure that the address is resolvable - // (this reduces the risk of exceptions when trying to connect to the service) - if (!address.endpoint().validateHost()) { - logger.warn("Omitting stale address {}, address does not resolve", address); - continue; - } - - ret.add(address); - - } - - return ret; - } - catch (Exception ex) { - return Set.of(); - } - } - - public Set> getGrpcEndpoints(ServiceId id, int node) { - try { - Set> ret = new HashSet<>(); - String restRoot = STR."/services/\{id.serviceName}/\{node}/grpc"; - for (var uuid : curatorFramework - .getChildren() - .forPath(restRoot)) { - - if (!isInstanceRunning(id, node, UUID.fromString(uuid))) { - continue; - } - - byte[] data = curatorFramework - .getData() - .forPath(ZKPaths.makePath(restRoot, uuid)); + long cxTime = curatorFramework.checkExists().forPath(path).getMzxid(); String hostAndPort = new String(data); - var address = GrpcEndpoint + var address = ServiceEndpoint .parse(hostAndPort) - .asInstance(UUID.fromString(uuid)); - - // Ensure that the address is resolvable - // (this reduces the risk of exceptions when trying to connect to the service) - if (!address.endpoint().validateHost()) { - logger.warn("Omitting stale address {}, address does not resolve", address); - continue; - } + .asInstance(UUID.fromString(uuid), cxTime); ret.add(address); - } return ret; @@ -254,29 +209,13 @@ public class ZkServiceRegistry implements ServiceRegistryIf { } public void registerMonitor(ServiceMonitorIf monitor) throws Exception { - monitor.register(this); - } + if (stopped) + logger.info("Not registering monitor for {} because the registry is stopped", monitor.getKey()); - public void registerMonitor(ServiceChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}"); - } + String path = monitor.getKey().toPath(); - public void registerMonitor(ServiceNodeChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}"); - } - - public void registerMonitor(ServiceRestEndpointChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/rest"); - } - - public void registerMonitor(ServiceGrpcEndpointChangeMonitor monitor) throws Exception { - installMonitor(monitor, STR."/services/\{monitor.serviceId.serviceName}/\{monitor.node}/grpc"); - } - - private void installMonitor(ServiceMonitorIf monitor, String path) throws Exception { - CuratorWatcher watcher = _ -> { + CuratorWatcher watcher = change -> { boolean reRegister; - try { reRegister = monitor.onChange(); } @@ -293,13 +232,31 @@ public class ZkServiceRegistry implements ServiceRegistryIf { curatorFramework.watchers().add() .usingWatcher(watcher) .forPath(path); + + // Also register for updates to the running-instances list, + // as this will have an effect on the result of getEndpoints() + curatorFramework.watchers().add() + .usingWatcher(watcher) + .forPath("/running-instances"); } /* Exposed for tests */ public synchronized void shutDown() { - if (!stopped) { - curatorFramework.close(); - stopped = true; + if (!stopped) + return; + + stopped = true; + + // Delete all liveness paths + for (var path : livenessPaths) { + logger.info("Cleaning up {}", path); + + try { + curatorFramework.delete().forPath(path); + } + catch (Exception ex) { + logger.error("Failed to delete path {}", path, ex); + } } } } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java index 63e35891..23faba58 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceChangeMonitor.java @@ -1,23 +1,17 @@ package nu.marginalia.service.discovery.monitor; -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; public abstract class ServiceChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; + public final ServiceKey serviceKey; - public ServiceChangeMonitor(ServiceId serviceId) { - this.serviceId = serviceId; + public ServiceChangeMonitor(ServiceKey key) { + this.serviceKey = key; } public abstract boolean onChange(); - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } + public ServiceKey getKey() { + return serviceKey; } + } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java deleted file mode 100644 index 2b33a40e..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceGrpcEndpointChangeMonitor.java +++ /dev/null @@ -1,25 +0,0 @@ -package nu.marginalia.service.discovery.monitor; - -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; - -public abstract class ServiceGrpcEndpointChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; - public final int node; - public ServiceGrpcEndpointChangeMonitor(ServiceId serviceId, int node) { - this.serviceId = serviceId; - this.node = node; - } - - public abstract boolean onChange(); - - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java index af4cabb8..8dca094a 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceMonitorIf.java @@ -1,16 +1,13 @@ package nu.marginalia.service.discovery.monitor; -import nu.marginalia.service.discovery.ServiceRegistryIf; + +import nu.marginalia.service.discovery.property.ServiceKey; public interface ServiceMonitorIf { /** Called when the monitored service has changed. * @return true if the monitor is to be refreshed */ boolean onChange(); + ServiceKey getKey(); - /** Register this monitor with the given registry. - * It is preferred to use {@link ServiceRegistryIf}'s - * registerMonitor function. - * */ - void register(ServiceRegistryIf registry) throws Exception; } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java deleted file mode 100644 index 42e04e89..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceNodeChangeMonitor.java +++ /dev/null @@ -1,25 +0,0 @@ -package nu.marginalia.service.discovery.monitor; - -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; - -public abstract class ServiceNodeChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; - public final int node; - public ServiceNodeChangeMonitor(ServiceId serviceId, int node) { - this.serviceId = serviceId; - this.node = node; - } - - public abstract boolean onChange(); - - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java deleted file mode 100644 index 1e8c8564..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/monitor/ServiceRestEndpointChangeMonitor.java +++ /dev/null @@ -1,25 +0,0 @@ -package nu.marginalia.service.discovery.monitor; - -import nu.marginalia.service.discovery.ServiceRegistryIf; -import nu.marginalia.service.discovery.ZkServiceRegistry; -import nu.marginalia.service.id.ServiceId; - -public abstract class ServiceRestEndpointChangeMonitor implements ServiceMonitorIf { - public final ServiceId serviceId; - public final int node; - public ServiceRestEndpointChangeMonitor(ServiceId serviceId, int node) { - this.serviceId = serviceId; - this.node = node; - } - - public abstract boolean onChange(); - - public void register(ServiceRegistryIf registry) throws Exception { - if (registry instanceof ZkServiceRegistry zkServiceRegistry) { - zkServiceRegistry.registerMonitor(this); - } - else { - registry.registerMonitor(this); - } - } -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java deleted file mode 100644 index dc8300af..00000000 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ApiSchema.java +++ /dev/null @@ -1,6 +0,0 @@ -package nu.marginalia.service.discovery.property; - -public enum ApiSchema { - REST, - GRPC -} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java new file mode 100644 index 00000000..64436bdd --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/PartitionTraits.java @@ -0,0 +1,8 @@ +package nu.marginalia.service.discovery.property; + +public interface PartitionTraits { + interface Grpc {}; + interface Unicast {}; + interface Multicast {}; + interface NoGrpc {}; +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java index 7763eca0..c52d335c 100644 --- a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceEndpoint.java @@ -1,17 +1,23 @@ package nu.marginalia.service.discovery.property; - -import lombok.SneakyThrows; - import java.net.*; import java.util.UUID; -public sealed interface ServiceEndpoint { - String host(); - int port(); +public record ServiceEndpoint(String host, int port) { - URL toURL(String endpoint, String query); - default InetSocketAddress toInetSocketAddress() { + public static ServiceEndpoint parse(String hostAndPort) { + var parts = hostAndPort.split(":"); + if (parts.length != 2) { + throw new IllegalArgumentException("Invalid host:port string: " + hostAndPort); + } + return new ServiceEndpoint(parts[0], Integer.parseInt(parts[1])); + } + + public URL toURL(String endpoint, String query) throws URISyntaxException, MalformedURLException { + return new URI("http", null, host, port, endpoint, query, null) + .toURL(); + } + public InetSocketAddress toInetSocketAddress() { return new InetSocketAddress(host(), port()); } @@ -19,7 +25,7 @@ public sealed interface ServiceEndpoint { * * @return true if the host is a valid */ - default boolean validateHost() { + public boolean validateHost() { try { // Throws UnknownHostException if the host is not a valid IP address or hostname // (this should not be slow since the DNS lookup should be local, and if it isn't; @@ -31,63 +37,11 @@ public sealed interface ServiceEndpoint { } } - static ServiceEndpoint forSchema(ApiSchema schema, String host, int port) { - return switch (schema) { - case REST -> new RestEndpoint(host, port); - case GRPC -> new GrpcEndpoint(host, port); - }; + public InstanceAddress asInstance(UUID instance, long cxTime) { + return new InstanceAddress(this, instance, cxTime); } - record RestEndpoint(String host, int port) implements ServiceEndpoint { - public static RestEndpoint parse(String hostColonPort) { - String[] parts = hostColonPort.split(":"); - - if (parts.length != 2) { - throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'"); - } - - return new RestEndpoint( - parts[0], - Integer.parseInt(parts[1]) - ); - } - - @SneakyThrows - public URL toURL(String endpoint, String query) { - return new URI("http", null, host, port, endpoint, query, null) - .toURL(); - } - - public InstanceAddress asInstance(UUID uuid) { - return new InstanceAddress<>(this, uuid); - } - } - - record GrpcEndpoint(String host, int port) implements ServiceEndpoint { - public static GrpcEndpoint parse(String hostColonPort) { - String[] parts = hostColonPort.split(":"); - - if (parts.length != 2) { - throw new IllegalArgumentException(STR."Invalid host:port-format '\{hostColonPort}'"); - } - - return new GrpcEndpoint( - parts[0], - Integer.parseInt(parts[1]) - ); - } - - public InstanceAddress asInstance(UUID uuid) { - return new InstanceAddress<>(this, uuid); - } - - @Override - public URL toURL(String endpoint, String query) { - throw new UnsupportedOperationException(); - } - } - - record InstanceAddress(T endpoint, UUID instance) { + public record InstanceAddress(ServiceEndpoint endpoint, UUID instance, long cxTime) { public String host() { return endpoint.host(); } diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java new file mode 100644 index 00000000..66ae5ded --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServiceKey.java @@ -0,0 +1,69 @@ +package nu.marginalia.service.discovery.property; + +import io.grpc.ServiceDescriptor; +import nu.marginalia.service.id.ServiceId; + +public sealed interface ServiceKey

{ + String toPath(); + + static ServiceKey forRest(ServiceId id) { + return new Rest(id.serviceName); + } + static ServiceKey forRest(ServiceId id, int node) { + if (node == 0) { + return forRest(id); + } + + return new Rest(id.serviceName + "-" + node); + } + + static Grpc forServiceDescriptor(ServiceDescriptor descriptor, ServicePartition partition) { + return new Grpc<>(descriptor.getName(), partition); + } + + static Grpc forGrpcApi(Class apiClass, P2 partition) { + try { + var name = apiClass.getField("SERVICE_NAME").get(null); + return new Grpc(name.toString(), partition); + } + catch (Exception e) { + throw new IllegalArgumentException("Could not get SERVICE_NAME from " + apiClass.getSimpleName(), e); + } + } + + + + Grpc forPartition(P2 partition); + + + record Rest(String name) implements ServiceKey { + public String toPath() { + return STR."/services/rest/\{name}"; + } + + @Override + public + + Grpc forPartition(P2 partition) + { + throw new UnsupportedOperationException(); + } + } + record Grpc

(String name, P partition) implements ServiceKey

{ + public String baseName() { + return STR."/services/grpc/\{name}"; + } + public String toPath() { + return STR."/services/grpc/\{name}/\{partition.identifier()}"; + } + + @Override + public + + Grpc forPartition(P2 partition) + { + return new Grpc<>(name, partition); + } + } + +} diff --git a/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java new file mode 100644 index 00000000..32aa37fb --- /dev/null +++ b/code/common/service-discovery/src/main/java/nu/marginalia/service/discovery/property/ServicePartition.java @@ -0,0 +1,29 @@ +package nu.marginalia.service.discovery.property; + +public sealed interface ServicePartition { + String identifier(); + + static Any any() { return new Any(); } + static Multi multi() { return new Multi(); } + static Partition partition(int node) { return new Partition(node); } + static None none() { return new None(); } + + record Any() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast { + public String identifier() { return "*"; } + + } + record Multi() implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Multicast { + public String identifier() { return "*"; } + + } + record Partition(int node) implements ServicePartition, PartitionTraits.Grpc, PartitionTraits.Unicast { + public String identifier() { + return Integer.toString(node); + } + + } + record None() implements ServicePartition, PartitionTraits.NoGrpc { + public String identifier() { return ""; } + + } +} diff --git a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java index 3684a75a..7963c857 100644 --- a/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java +++ b/code/common/service-discovery/src/test/java/nu/marginalia/service/discovery/ZkServiceRegistryTest.java @@ -1,6 +1,8 @@ package nu.marginalia.service.discovery; -import nu.marginalia.service.discovery.property.ApiSchema; +import nu.marginalia.api.math.MathApiGrpc; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -13,7 +15,6 @@ import org.testcontainers.junit.jupiter.Testcontainers; import java.util.*; -import static nu.marginalia.service.discovery.property.ServiceEndpoint.*; import static org.junit.jupiter.api.Assertions.*; @Testcontainers @@ -58,15 +59,18 @@ class ZkServiceRegistryTest { List ports = new ArrayList<>(); Set portsSet = new HashSet<>(); + + var key = ServiceKey.forRest(ServiceId.Search, 0); + for (int i = 0; i < 500; i++) { - int port = registry1.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0); + int port = registry1.requestPort("127.0.0.1", key); ports.add(port); // Ensure we get unique ports assertTrue(portsSet.add(port)); } for (int i = 0; i < 50; i++) { - int port = registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0); + int port = registry2.requestPort("127.0.0.1", key); ports.add(port); // Ensure we get unique ports @@ -75,39 +79,86 @@ class ZkServiceRegistryTest { registry1.shutDown(); for (int i = 0; i < 500; i++) { // Verify we can reclaim ports - ports.add(registry2.requestPort("127.0.0.1", ApiSchema.REST, ServiceId.Search, 0)); + ports.add(registry2.requestPort("127.0.0.1", key)); } assertEquals(1050, ports.size()); } @Test - void getInstances() throws Exception { + void getInstancesRestgRPC() throws Exception { var uuid1 = UUID.randomUUID(); var uuid2 = UUID.randomUUID(); var registry1 = createRegistry(); var registry2 = createRegistry(); - var endpoint1 = (RestEndpoint) registry1.registerService(ApiSchema.REST, ServiceId.Search, 0, uuid1, "127.0.0.1"); - var endpoint2 = (GrpcEndpoint) registry2.registerService(ApiSchema.GRPC, ServiceId.Search, 0, uuid2, "127.0.0.2"); + var key1 = ServiceKey.forRest(ServiceId.Search, 0); + var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()); - registry1.announceInstance(ServiceId.Search, 0, uuid1); - registry2.announceInstance(ServiceId.Search, 0, uuid2); + var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1"); + var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2"); - assertEquals(Set.of(endpoint1.asInstance(uuid1)), - registry1.getRestEndpoints(ServiceId.Search, 0)); + registry1.announceInstance(uuid1); + registry2.announceInstance(uuid2); - assertEquals(Set.of(endpoint2.asInstance(uuid2)), - registry1.getGrpcEndpoints(ServiceId.Search, 0)); + assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), + registry1.getEndpoints(key1)); + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), + registry1.getEndpoints(key2)); registry1.shutDown(); Thread.sleep(100); - assertEquals(Set.of(), - registry2.getRestEndpoints(ServiceId.Search, 0)); - assertEquals(Set.of(endpoint2.asInstance(uuid2)), - registry2.getGrpcEndpoints(ServiceId.Search, 0)); + assertEquals(Set.of(), registry2.getEndpoints(key1)); + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key2)); + } + + @Test + void testInstancesTwoAny() throws Exception { + var uuid1 = UUID.randomUUID(); + var uuid2 = UUID.randomUUID(); + + var registry1 = createRegistry(); + var registry2 = createRegistry(); + + var key = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()); + + var endpoint1 = registry1.registerService(key, uuid1, "127.0.0.1"); + var endpoint2 = registry2.registerService(key, uuid2, "127.0.0.2"); + + registry1.announceInstance(uuid1); + registry2.announceInstance(uuid2); + + assertEquals(Set.of(endpoint1.asInstance(uuid1, 0), + endpoint2.asInstance(uuid2, 0)), + registry1.getEndpoints(key)); + + registry1.shutDown(); + Thread.sleep(100); + + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry2.getEndpoints(key)); + } + + @Test + void testInstancesTwoPartitions() throws Exception { + var uuid1 = UUID.randomUUID(); + var uuid2 = UUID.randomUUID(); + + var registry1 = createRegistry(); + var registry2 = createRegistry(); + + var key1 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(1)); + var key2 = ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.partition(2)); + + var endpoint1 = registry1.registerService(key1, uuid1, "127.0.0.1"); + var endpoint2 = registry2.registerService(key2, uuid2, "127.0.0.2"); + + registry1.announceInstance(uuid1); + registry2.announceInstance(uuid2); + + assertEquals(Set.of(endpoint1.asInstance(uuid1, 0)), registry1.getEndpoints(key1)); + assertEquals(Set.of(endpoint2.asInstance(uuid2, 0)), registry1.getEndpoints(key2)); } @Test @@ -115,9 +166,9 @@ class ZkServiceRegistryTest { var registry1 = createRegistry(); var uuid1 = UUID.randomUUID(); - assertFalse(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1)); - registry1.announceInstance(ServiceId.Search, 0, uuid1); - assertTrue(registry1.isInstanceRunning(ServiceId.Search, 0, uuid1)); + assertFalse(registry1.isInstanceRunning(uuid1)); + registry1.announceInstance(uuid1); + assertTrue(registry1.isInstanceRunning(uuid1)); registry1.shutDown(); } diff --git a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java index 800d6712..4a6f1c71 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java +++ b/code/common/service/src/main/java/nu/marginalia/service/control/ServiceHeartbeatImpl.java @@ -156,6 +156,8 @@ public class ServiceHeartbeatImpl implements ServiceHeartbeat { stmt.executeUpdate(); } } + + dataSource.close(); } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java index aa8b1203..a196872f 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java @@ -6,6 +6,7 @@ import com.google.inject.Singleton; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; +import nu.marginalia.WmsaHome; import nu.marginalia.service.ServiceHomeNotConfiguredException; import org.flywaydb.core.Flyway; import org.mariadb.jdbc.Driver; @@ -51,7 +52,7 @@ public class DatabaseModule extends AbstractModule { } private Properties loadDbProperties() { - Path propDir = getHomePath().resolve("conf/db.properties"); + Path propDir = WmsaHome.getHomePath().resolve("conf/db.properties"); if (!Files.isRegularFile(propDir)) { throw new IllegalStateException("Database properties file " + propDir + " does not exist"); } @@ -72,17 +73,6 @@ public class DatabaseModule extends AbstractModule { } - public static Path getHomePath() { - var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse("/var/lib/wmsa"); - - var ret = Path.of(retStr); - if (!Files.isDirectory(ret)) { - throw new ServiceHomeNotConfiguredException("Could not find WMSA_HOME, either set environment variable or ensure /var/lib/wmsa exists"); - } - return ret; - } - - @SneakyThrows @Singleton @Provides @@ -97,7 +87,6 @@ public class DatabaseModule extends AbstractModule { try { HikariConfig config = new HikariConfig(); - config.setJdbcUrl(connStr); config.setUsername(dbProperties.getProperty(DB_USER_KEY)); config.setPassword(dbProperties.getProperty(DB_PASS_KEY)); diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java index e64c88a8..cf152bf0 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfiguration.java @@ -1,5 +1,6 @@ package nu.marginalia.service.module; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.id.ServiceId; import java.util.UUID; diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java index 37ce99d5..8ed7f45c 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/ServiceConfigurationModule.java @@ -65,7 +65,7 @@ public class ServiceConfigurationModule extends AbstractModule { } // If we're in docker, we'll use the hostname - if (isDocker()) { + if (Boolean.getBoolean("service.useDockerHostname")) { return System.getenv("HOSTNAME"); } @@ -82,14 +82,7 @@ public class ServiceConfigurationModule extends AbstractModule { return configuredValue; } - // If we're in docker, we'll bind to all interfaces - if (isDocker()) - return "0.0.0.0"; - else // If we're not in docker, we'll default to binding to localhost to avoid exposing services - return "127.0.0.1"; + return "127.0.0.1"; } - boolean isDocker() { - return System.getenv("WMSA_IN_DOCKER") != null; - } } diff --git a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java index b9bb082b..ba076ae5 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/server/Service.java +++ b/code/common/service/src/main/java/nu/marginalia/service/server/Service.java @@ -5,8 +5,8 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.prometheus.client.Counter; import lombok.SneakyThrows; import nu.marginalia.mq.inbox.*; -import nu.marginalia.service.discovery.property.ApiSchema; -import nu.marginalia.service.discovery.property.ServiceEndpoint; +import nu.marginalia.service.discovery.property.*; +import nu.marginalia.service.id.ServiceId; import nu.marginalia.service.server.mq.ServiceMqSubscription; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,6 +16,7 @@ import spark.Request; import spark.Response; import spark.Spark; +import java.net.InetSocketAddress; import java.util.List; import java.util.Optional; @@ -48,11 +49,24 @@ public class Service { @SneakyThrows public Service(BaseServiceParams params, Runnable configureStaticFiles, + ServicePartition partition, List grpcServices) { + this.initialization = params.initialization; var config = params.configuration; node = config.node(); + if (config.serviceId() == ServiceId.Control) { + // Special case for first boot, since the control service + // owns database migrations and so on, we need other processes + // to wait for this to be done before they start. This is + // only needed once. + params.serviceRegistry.declareFirstBoot(); + } + else { + params.serviceRegistry.waitForFirstBoot(); + } + String inboxName = config.serviceName(); logger.info("Inbox name: {}", inboxName); @@ -60,8 +74,7 @@ public class Service { var restEndpoint = serviceRegistry.registerService( - ApiSchema.REST, config.serviceId(), - config.node(), + ServiceKey.forRest(config.serviceId(), config.node()), config.instanceUuid(), config.externalAddress() ); @@ -75,7 +88,7 @@ public class Service { initialization.addCallback(params.heartbeat::start); initialization.addCallback(messageQueueInbox::start); initialization.addCallback(() -> params.eventLog.logEvent("SVC-INIT", serviceName + ":" + config.node())); - initialization.addCallback(() -> serviceRegistry.announceInstance(config.serviceId(), config.node(), config.instanceUuid())); + initialization.addCallback(() -> serviceRegistry.announceInstance(config.instanceUuid())); if (!initialization.isReady() && ! initialized ) { initialized = true; @@ -101,29 +114,39 @@ public class Service { Spark.get("/internal/started", this::isInitialized); Spark.get("/internal/ready", this::isReady); - ServiceEndpoint.GrpcEndpoint grpcEndpoint = (ServiceEndpoint.GrpcEndpoint) params.serviceRegistry.registerService( - ApiSchema.GRPC, config.serviceId(), - config.node(), - config.instanceUuid(), - config.externalAddress() - ); + int port = params.serviceRegistry.requestPort(config.externalAddress(), new ServiceKey.Grpc<>("-", partition)); // Start the gRPC server - var grpcServerBuilder = NettyServerBuilder.forAddress(grpcEndpoint.toInetSocketAddress()); + var grpcServerBuilder = NettyServerBuilder.forAddress(new InetSocketAddress(config.bindAddress(), port)); for (var grpcService : grpcServices) { - grpcServerBuilder.addService(grpcService); + var svc = grpcService.bindService(); + + params.serviceRegistry.registerService( + ServiceKey.forServiceDescriptor(svc.getServiceDescriptor(), partition), + config.instanceUuid(), + config.externalAddress() + ); + + grpcServerBuilder.addService(svc); } grpcServerBuilder.build().start(); } } public Service(BaseServiceParams params, + ServicePartition partition, List grpcServices) { - this(params, Service::defaultSparkConfig, grpcServices); + this(params, + Service::defaultSparkConfig, + partition, + grpcServices); } public Service(BaseServiceParams params) { - this(params, Service::defaultSparkConfig, List.of()); + this(params, + Service::defaultSparkConfig, + ServicePartition.any(), + List.of()); } private static void defaultSparkConfig() { diff --git a/code/common/service/src/main/resources/log4j2-prod.xml b/code/common/service/src/main/resources/log4j2-prod.xml index 43150a2e..01c914ac 100644 --- a/code/common/service/src/main/resources/log4j2-prod.xml +++ b/code/common/service/src/main/resources/log4j2-prod.xml @@ -21,6 +21,8 @@ + + diff --git a/code/common/service/src/main/resources/log4j2-test.xml b/code/common/service/src/main/resources/log4j2-test.xml index 3158f632..8fd0b262 100644 --- a/code/common/service/src/main/resources/log4j2-test.xml +++ b/code/common/service/src/main/resources/log4j2-test.xml @@ -20,6 +20,8 @@ + + diff --git a/code/features-index/domain-ranking/build.gradle b/code/features-index/domain-ranking/build.gradle index fc8e8401..42af2c69 100644 --- a/code/features-index/domain-ranking/build.gradle +++ b/code/features-index/domain-ranking/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':code:common:db') implementation project(':code:common:model') implementation project(':code:common:service') - implementation project(':code:api:query-api') + implementation project(':code:functions:domain-links:api') implementation 'org.jgrapht:jgrapht-core:1.5.2' diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java index 9d5564d0..f4adf2b6 100644 --- a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java +++ b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/InvertedLinkGraphSource.java @@ -3,23 +3,20 @@ package nu.marginalia.ranking.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; import org.jgrapht.graph.DefaultEdge; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; /** A source for the inverted link graph, * which is the same as the regular graph except * the direction of the links have been inverted */ public class InvertedLinkGraphSource extends AbstractGraphSource { - private final QueryClient queryClient; + private final AggregateDomainLinksClient queryClient; @Inject - public InvertedLinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) { + public InvertedLinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient queryClient) { super(dataSource); this.queryClient = queryClient; } diff --git a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java index cc7f2b53..5367a570 100644 --- a/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java +++ b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/LinkGraphSource.java @@ -3,19 +3,19 @@ package nu.marginalia.ranking.data; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import org.jgrapht.Graph; import org.jgrapht.graph.DefaultDirectedGraph; import org.jgrapht.graph.DefaultEdge; /** A source for the regular link graph. */ public class LinkGraphSource extends AbstractGraphSource { - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; @Inject - public LinkGraphSource(HikariDataSource dataSource, QueryClient queryClient) { + public LinkGraphSource(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) { super(dataSource); - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; } @SneakyThrows @@ -25,7 +25,7 @@ public class LinkGraphSource extends AbstractGraphSource { addVertices(graph); - var allLinks = queryClient.getAllDomainLinks(); + var allLinks = domainLinksClient.getAllDomainLinks(); var iter = allLinks.iterator(); while (iter.advance()) { if (!graph.containsVertex(iter.dest())) { diff --git a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java b/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java index 7fdd2f82..b92543e2 100644 --- a/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java +++ b/code/features-index/domain-ranking/src/test/java/nu/marginalia/ranking/RankingAlgorithmsContainerTest.java @@ -3,7 +3,7 @@ package nu.marginalia.ranking; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.ranking.data.InvertedLinkGraphSource; import nu.marginalia.ranking.data.LinkGraphSource; import nu.marginalia.ranking.data.SimilarityGraphSource; @@ -37,8 +37,9 @@ public class RankingAlgorithmsContainerTest { static HikariDataSource dataSource; - QueryClient queryClient; - QueryClient.AllLinks allLinks; + AggregateDomainLinksClient domainLinksClient; + AggregateDomainLinksClient.AllLinks allLinks; + @BeforeAll public static void setup() { HikariConfig config = new HikariConfig(); @@ -66,9 +67,9 @@ public class RankingAlgorithmsContainerTest { @BeforeEach public void setupQueryClient() { - queryClient = Mockito.mock(QueryClient.class); - allLinks = new QueryClient.AllLinks(); - when(queryClient.getAllDomainLinks()).thenReturn(allLinks); + domainLinksClient = Mockito.mock(AggregateDomainLinksClient.class); + allLinks = new AggregateDomainLinksClient.AllLinks(); + when(domainLinksClient.getAllDomainLinks()).thenReturn(allLinks); try (var conn = dataSource.getConnection(); var stmt = conn.createStatement()) { @@ -97,7 +98,7 @@ public class RankingAlgorithmsContainerTest { @Test public void testGetDomains() { // should all be the same, doesn't matter which one we use - var source = new LinkGraphSource(dataSource, queryClient); + var source = new LinkGraphSource(dataSource, domainLinksClient); Assertions.assertEquals(List.of(1), source.domainIds(List.of("memex.marginalia.nu"))); @@ -111,7 +112,7 @@ public class RankingAlgorithmsContainerTest { public void testLinkGraphSource() { allLinks.add(1, 3); - var graph = new LinkGraphSource(dataSource, queryClient).getGraph(); + var graph = new LinkGraphSource(dataSource, domainLinksClient).getGraph(); Assertions.assertTrue(graph.containsVertex(1)); Assertions.assertTrue(graph.containsVertex(2)); @@ -127,7 +128,7 @@ public class RankingAlgorithmsContainerTest { public void testInvertedLinkGraphSource() { allLinks.add(1, 3); - var graph = new InvertedLinkGraphSource(dataSource, queryClient).getGraph(); + var graph = new InvertedLinkGraphSource(dataSource, domainLinksClient).getGraph(); Assertions.assertTrue(graph.containsVertex(1)); Assertions.assertTrue(graph.containsVertex(2)); diff --git a/code/functions/domain-info/api/build.gradle b/code/functions/domain-info/api/build.gradle new file mode 100644 index 00000000..74c0548e --- /dev/null +++ b/code/functions/domain-info/api/build.gradle @@ -0,0 +1,45 @@ +plugins { + id 'java' + + id "com.google.protobuf" version "0.9.4" + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +jar.archiveBaseName = 'domain-info-api' + +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + } +} + +apply from: "$rootProject.projectDir/protobuf.gradle" + +dependencies { + implementation project(':code:common:model') + implementation project(':code:common:config') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.notnull + implementation libs.guice + implementation libs.gson + implementation libs.protobuf + implementation libs.javax.annotation + implementation libs.bundles.grpc + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + +} diff --git a/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java new file mode 100644 index 00000000..ee305994 --- /dev/null +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainInfoClient.java @@ -0,0 +1,56 @@ +package nu.marginalia.api.domains; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.api.domains.model.SimilarDomain; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.*; + +import nu.marginalia.api.domains.model.*; + +@Singleton +public class DomainInfoClient { + private static final Logger logger = LoggerFactory.getLogger(DomainInfoClient.class); + + private final GrpcSingleNodeChannelPool channelPool; + private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); + + @Inject + public DomainInfoClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createSingle( + ServiceKey.forGrpcApi(DomainInfoAPIGrpc.class, ServicePartition.any()), + DomainInfoAPIGrpc::newBlockingStub); + } + + public Future> similarDomains(int domainId, int count) { + return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getSimilarDomains) + .async(virtualExecutorService) + .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count)) + .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse); + } + + public Future> linkedDomains(int domainId, int count) { + return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getLinkingDomains) + .async(virtualExecutorService) + .run(DomainsProtobufCodec.DomainQueries.createRequest(domainId, count)) + .thenApply(DomainsProtobufCodec.DomainQueries::convertResponse); + } + + public Future domainInformation(int domainId) { + return channelPool.call(DomainInfoAPIGrpc.DomainInfoAPIBlockingStub::getDomainInfo) + .async(virtualExecutorService) + .run(DomainsProtobufCodec.DomainInfo.createRequest(domainId)) + .thenApply(DomainsProtobufCodec.DomainInfo::convertResponse); + } + + public boolean isAccepting() { + return channelPool.hasChannel(); + } +} diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java similarity index 51% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java rename to code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java index cd2d61fa..d3c90f9f 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/AssistantProtobufCodec.java +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/DomainsProtobufCodec.java @@ -1,74 +1,14 @@ -package nu.marginalia.assistant.client; +package nu.marginalia.api.domains; import lombok.SneakyThrows; -import nu.marginalia.assistant.api.*; -import nu.marginalia.assistant.client.model.DictionaryEntry; -import nu.marginalia.assistant.client.model.DictionaryResponse; -import nu.marginalia.assistant.client.model.DomainInformation; -import nu.marginalia.assistant.client.model.SimilarDomain; import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.EdgeUrl; +import nu.marginalia.api.domains.model.*; import java.util.ArrayList; import java.util.List; -public class AssistantProtobufCodec { - - public static class DictionaryLookup { - public static RpcDictionaryLookupRequest createRequest(String word) { - return RpcDictionaryLookupRequest.newBuilder() - .setWord(word) - .build(); - } - public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) { - return new DictionaryResponse( - rsp.getWord(), - rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList() - ); - } - - private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) { - return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition()); - } - } - - public static class SpellCheck { - public static RpcSpellCheckRequest createRequest(String text) { - return RpcSpellCheckRequest.newBuilder() - .setText(text) - .build(); - } - - public static List convertResponse(RpcSpellCheckResponse rsp) { - return rsp.getSuggestionsList(); - } - } - - public static class UnitConversion { - public static RpcUnitConversionRequest createRequest(String from, String to, String unit) { - return RpcUnitConversionRequest.newBuilder() - .setFrom(from) - .setTo(to) - .setUnit(unit) - .build(); - } - - public static String convertResponse(RpcUnitConversionResponse rsp) { - return rsp.getResult(); - } - } - - public static class EvalMath { - public static RpcEvalMathRequest createRequest(String expression) { - return RpcEvalMathRequest.newBuilder() - .setExpression(expression) - .build(); - } - - public static String convertResponse(RpcEvalMathResponse rsp) { - return rsp.getResult(); - } - } +public class DomainsProtobufCodec { public static class DomainQueries { public static RpcDomainLinksRequest createRequest(int domainId, int count) { diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DomainInformation.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java similarity index 96% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DomainInformation.java rename to code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java index 5be64a97..2e62af22 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DomainInformation.java +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/DomainInformation.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.domains.model; import lombok.*; import nu.marginalia.model.EdgeDomain; diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/SimilarDomain.java b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java similarity index 97% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/SimilarDomain.java rename to code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java index 1bdae22e..d15d0f3d 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/SimilarDomain.java +++ b/code/functions/domain-info/api/src/main/java/nu/marginalia/api/domains/model/SimilarDomain.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.domains.model; import nu.marginalia.model.EdgeUrl; diff --git a/code/api/assistant-api/src/main/protobuf/assistant-api.proto b/code/functions/domain-info/api/src/main/protobuf/domain-info.proto similarity index 77% rename from code/api/assistant-api/src/main/protobuf/assistant-api.proto rename to code/functions/domain-info/api/src/main/protobuf/domain-info.proto index 12aa7ce1..8be12d11 100644 --- a/code/api/assistant-api/src/main/protobuf/assistant-api.proto +++ b/code/functions/domain-info/api/src/main/protobuf/domain-info.proto @@ -1,18 +1,10 @@ syntax="proto3"; -package assistantapi; +package marginalia.api.domain; -option java_package="nu.marginalia.assistant.api"; +option java_package="nu.marginalia.api.domains"; option java_multiple_files=true; -service AssistantApi { - /** Looks up a word in the dictionary. */ - rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {} - /** Checks the spelling of a text. */ - rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {} - /** Converts a unit from one to another. */ - rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {} - /** Evaluates a mathematical expression. */ - rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {} +service DomainInfoAPI { /** Fetches information about a domain. */ rpc getDomainInfo(RpcDomainId) returns (RpcDomainInfoResponse) {} diff --git a/code/functions/domain-info/build.gradle b/code/functions/domain-info/build.gradle new file mode 100644 index 00000000..c9f641d0 --- /dev/null +++ b/code/functions/domain-info/build.gradle @@ -0,0 +1,44 @@ +plugins { + id 'java' + + id 'application' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':code:functions:domain-info:api') + implementation project(':code:functions:domain-links:api') + + implementation project(':code:common:config') + implementation project(':code:common:service') + implementation project(':code:common:model') + implementation project(':code:common:db') + implementation project(':code:common:service-discovery') + + implementation project(':code:libraries:geo-ip') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.spark + implementation libs.opencsv + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + + +} diff --git a/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java new file mode 100644 index 00000000..9bf8a1c2 --- /dev/null +++ b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInfoGrpcService.java @@ -0,0 +1,53 @@ +package nu.marginalia.functions.domains; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import nu.marginalia.api.domains.DomainInfoAPIGrpc; +import nu.marginalia.api.domains.*; + +public class DomainInfoGrpcService extends DomainInfoAPIGrpc.DomainInfoAPIImplBase { + + private final DomainInformationService domainInformationService; + private final SimilarDomainsService similarDomainsService; + @Inject + public DomainInfoGrpcService(DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService) + { + + this.domainInformationService = domainInformationService; + this.similarDomainsService = similarDomainsService; + } + + @Override + public void getDomainInfo(RpcDomainId request, StreamObserver responseObserver) { + var ret = domainInformationService.domainInfo(request.getDomainId()); + + ret.ifPresent(responseObserver::onNext); + + responseObserver.onCompleted(); + } + + @Override + public void getSimilarDomains(RpcDomainLinksRequest request, + StreamObserver responseObserver) { + var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount()); + + var responseBuilder = RpcSimilarDomains + .newBuilder() + .addAllDomains(ret); + + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver responseObserver) { + var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount()); + + var responseBuilder = RpcSimilarDomains + .newBuilder() + .addAllDomains(ret); + + responseObserver.onNext(responseBuilder.build()); + responseObserver.onCompleted(); + } +} diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java similarity index 87% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java rename to code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java index f2590e7e..bf8ad9df 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/DomainInformationService.java +++ b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/DomainInformationService.java @@ -1,11 +1,11 @@ -package nu.marginalia.assistant.domains; +package nu.marginalia.functions.domains; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.assistant.api.RpcDomainInfoResponse; +import nu.marginalia.api.domains.RpcDomainInfoResponse; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.geoip.GeoIpDictionary; import nu.marginalia.model.EdgeDomain; import nu.marginalia.db.DbDomainQueries; -import nu.marginalia.query.client.QueryClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +21,7 @@ public class DomainInformationService { private final GeoIpDictionary geoIpDictionary; private DbDomainQueries dbDomainQueries; - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; private HikariDataSource dataSource; private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -29,11 +29,11 @@ public class DomainInformationService { public DomainInformationService( DbDomainQueries dbDomainQueries, GeoIpDictionary geoIpDictionary, - QueryClient queryClient, + AggregateDomainLinksClient domainLinksClient, HikariDataSource dataSource) { this.dbDomainQueries = dbDomainQueries; this.geoIpDictionary = geoIpDictionary; - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; this.dataSource = dataSource; } @@ -84,8 +84,8 @@ public class DomainInformationService { inCrawlQueue = rs.next(); builder.setInCrawlQueue(inCrawlQueue); - builder.setIncomingLinks(queryClient.countLinksToDomain(domainId)); - builder.setOutboundLinks(queryClient.countLinksFromDomain(domainId)); + builder.setIncomingLinks(domainLinksClient.countLinksToDomain(domainId)); + builder.setOutboundLinks(domainLinksClient.countLinksFromDomain(domainId)); rs = stmt.executeQuery(STR.""" SELECT KNOWN_URLS, GOOD_URLS, VISITED_URLS FROM DOMAIN_METADATA WHERE ID=\{domainId} diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java similarity index 95% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java rename to code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java index 4d00864f..8ce2b55c 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/domains/SimilarDomainsService.java +++ b/code/functions/domain-info/src/main/java/nu/marginalia/functions/domains/SimilarDomainsService.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.domains; +package nu.marginalia.functions.domains; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; @@ -8,10 +8,10 @@ import gnu.trove.map.hash.TIntDoubleHashMap; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; -import nu.marginalia.assistant.api.RpcSimilarDomain; -import nu.marginalia.assistant.client.model.SimilarDomain; +import nu.marginalia.api.domains.*; +import nu.marginalia.api.domains.model.SimilarDomain; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.model.EdgeDomain; -import nu.marginalia.query.client.QueryClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,7 +27,7 @@ public class SimilarDomainsService { private static final Logger logger = LoggerFactory.getLogger(SimilarDomainsService.class); private final HikariDataSource dataSource; - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; private volatile TIntIntHashMap domainIdToIdx = new TIntIntHashMap(100_000); private volatile int[] domainIdxToId; @@ -43,9 +43,9 @@ public class SimilarDomainsService { volatile boolean isReady = false; @Inject - public SimilarDomainsService(HikariDataSource dataSource, QueryClient queryClient) { + public SimilarDomainsService(HikariDataSource dataSource, AggregateDomainLinksClient domainLinksClient) { this.dataSource = dataSource; - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; Executors.newSingleThreadExecutor().submit(this::init); } @@ -256,7 +256,7 @@ public class SimilarDomainsService { private TIntSet getLinkingIdsDToS(int domainIdx) { var items = new TIntHashSet(); - for (int id : queryClient.getLinksFromDomain(domainIdxToId[domainIdx])) { + for (int id : domainLinksClient.getLinksFromDomain(domainIdxToId[domainIdx])) { items.add(domainIdToIdx.get(id)); } @@ -266,7 +266,7 @@ public class SimilarDomainsService { private TIntSet getLinkingIdsSToD(int domainIdx) { var items = new TIntHashSet(); - for (int id : queryClient.getLinksToDomain(domainIdxToId[domainIdx])) { + for (int id : domainLinksClient.getLinksToDomain(domainIdxToId[domainIdx])) { items.add(domainIdToIdx.get(id)); } diff --git a/code/functions/domain-links/aggregate/build.gradle b/code/functions/domain-links/aggregate/build.gradle new file mode 100644 index 00000000..b8702030 --- /dev/null +++ b/code/functions/domain-links/aggregate/build.gradle @@ -0,0 +1,36 @@ +plugins { + id 'java' + + id 'application' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':code:functions:domain-links:api') + + implementation project(':code:common:config') + implementation project(':code:common:service') + implementation project(':code:common:model') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.fastutil + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + + +} diff --git a/code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java b/code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java new file mode 100644 index 00000000..ff842075 --- /dev/null +++ b/code/functions/domain-links/aggregate/src/main/java/nu/marginalia/functions/domainlinks/AggregateDomainLinksService.java @@ -0,0 +1,96 @@ +package nu.marginalia.functions.domainlinks; + +import com.google.inject.Inject; +import io.grpc.stub.StreamObserver; +import nu.marginalia.api.domainlink.*; +import nu.marginalia.api.indexdomainlinks.PartitionDomainLinksClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class AggregateDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase { + private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksService.class); + private final PartitionDomainLinksClient client; + + @Inject + public AggregateDomainLinksService(PartitionDomainLinksClient client) { + this.client = client; + } + + @Override + public void getAllLinks(Empty request, + StreamObserver responseObserver) { + + client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks) + .run(Empty.getDefaultInstance()) + .forEach(iter -> iter.forEachRemaining(responseObserver::onNext)); + + responseObserver.onCompleted(); + } + + @Override + public void getLinksFromDomain(RpcDomainId request, + StreamObserver responseObserver) { + var rspBuilder = RpcDomainIdList.newBuilder(); + + client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain) + .run(request) + .stream() + .map(RpcDomainIdList::getDomainIdList) + .flatMap(List::stream) + .forEach(rspBuilder::addDomainId); + + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void getLinksToDomain(RpcDomainId request, + StreamObserver responseObserver) { + var rspBuilder = RpcDomainIdList.newBuilder(); + + + client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain) + .run(request) + .stream() + .map(RpcDomainIdList::getDomainIdList) + .flatMap(List::stream) + .forEach(rspBuilder::addDomainId); + + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void countLinksFromDomain(RpcDomainId request, + StreamObserver responseObserver) { + int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain) + .run(request) + .stream() + .mapToInt(RpcDomainIdCount::getIdCount) + .sum(); + + var rspBuilder = RpcDomainIdCount.newBuilder(); + rspBuilder.setIdCount(sum); + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + + @Override + public void countLinksToDomain(RpcDomainId request, + StreamObserver responseObserver) { + + int sum = client.getChannelPool().call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain) + .run(request) + .stream() + .mapToInt(RpcDomainIdCount::getIdCount) + .sum(); + + var rspBuilder = RpcDomainIdCount.newBuilder(); + rspBuilder.setIdCount(sum); + responseObserver.onNext(rspBuilder.build()); + responseObserver.onCompleted(); + } + +} diff --git a/code/functions/domain-links/api/build.gradle b/code/functions/domain-links/api/build.gradle new file mode 100644 index 00000000..081e3167 --- /dev/null +++ b/code/functions/domain-links/api/build.gradle @@ -0,0 +1,46 @@ +plugins { + id 'java' + + id "com.google.protobuf" version "0.9.4" + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +jar.archiveBaseName = 'index-domain-links-api' + +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + } +} + +apply from: "$rootProject.projectDir/protobuf.gradle" + +dependencies { + implementation project(':code:common:model') + implementation project(':code:common:config') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.notnull + implementation libs.guice + implementation libs.gson + implementation libs.protobuf + implementation libs.roaringbitmap + implementation libs.javax.annotation + implementation libs.bundles.grpc + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + +} diff --git a/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java new file mode 100644 index 00000000..c8f5c5ec --- /dev/null +++ b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/AggregateDomainLinksClient.java @@ -0,0 +1,138 @@ +package nu.marginalia.api.indexdomainlinks; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.api.domainlink.DomainLinksApiGrpc; +import nu.marginalia.api.domainlink.Empty; +import nu.marginalia.api.domainlink.RpcDomainId; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.roaringbitmap.longlong.PeekableLongIterator; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.List; + +@Singleton +public class AggregateDomainLinksClient { + private static final Logger logger = LoggerFactory.getLogger(AggregateDomainLinksClient.class); + + private final GrpcSingleNodeChannelPool channelPool; + + @Inject + public AggregateDomainLinksClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createSingle( + ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.any()), + DomainLinksApiGrpc::newBlockingStub); + } + + + public AllLinks getAllDomainLinks() { + AllLinks links = new AllLinks(); + + channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getAllLinks) + .run(Empty.getDefaultInstance()) + .forEachRemaining(pairs -> { + for (int i = 0; i < pairs.getDestIdsCount(); i++) { + links.add(pairs.getSourceIds(i), pairs.getDestIds(i)); + } + }); + + return links; + } + + public List getLinksToDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksToDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getDomainIdList() + .stream() + .sorted() + .toList(); + } + catch (Exception e) { + logger.error("API Exception", e); + return List.of(); + } + } + + public List getLinksFromDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::getLinksFromDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getDomainIdList() + .stream() + .sorted() + .toList(); + + } + catch (Exception e) { + logger.error("API Exception", e); + return List.of(); + } + } + + public int countLinksToDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksToDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getIdCount(); + + } + catch (Exception e) { + logger.error("API Exception", e); + return 0; + } + } + + public int countLinksFromDomain(int domainId) { + try { + return channelPool.call(DomainLinksApiGrpc.DomainLinksApiBlockingStub::countLinksFromDomain) + .run(RpcDomainId.newBuilder().setDomainId(domainId).build()) + .getIdCount(); + } + catch (Exception e) { + logger.error("API Exception", e); + return 0; + } + } + + public boolean waitReady(Duration duration) throws InterruptedException { + return channelPool.awaitChannel(duration); + } + + public static class AllLinks { + private final Roaring64Bitmap sourceToDest = new Roaring64Bitmap(); + + public void add(int source, int dest) { + sourceToDest.add(Integer.toUnsignedLong(source) << 32 | Integer.toUnsignedLong(dest)); + } + + public Iterator iterator() { + return new Iterator(); + } + + public class Iterator { + private final PeekableLongIterator base = sourceToDest.getLongIterator(); + long val = Long.MIN_VALUE; + + public boolean advance() { + if (base.hasNext()) { + val = base.next(); + return true; + } + return false; + } + public int source() { + return (int) (val >>> 32); + } + public int dest() { + return (int) (val & 0xFFFF_FFFFL); + } + } + } +} diff --git a/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java new file mode 100644 index 00000000..6e561b12 --- /dev/null +++ b/code/functions/domain-links/api/src/main/java/nu/marginalia/api/indexdomainlinks/PartitionDomainLinksClient.java @@ -0,0 +1,30 @@ +package nu.marginalia.api.indexdomainlinks; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.api.domainlink.DomainLinksApiGrpc; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcMultiNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class PartitionDomainLinksClient { + private static final Logger logger = LoggerFactory.getLogger(PartitionDomainLinksClient.class); + + private final GrpcMultiNodeChannelPool channelPool; + + @Inject + public PartitionDomainLinksClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createMulti( + ServiceKey.forGrpcApi(DomainLinksApiGrpc.class, ServicePartition.multi()), + DomainLinksApiGrpc::newBlockingStub); + } + + public GrpcMultiNodeChannelPool getChannelPool() { + return channelPool; + } + +} diff --git a/code/functions/domain-links/api/src/main/protobuf/domain-links.proto b/code/functions/domain-links/api/src/main/protobuf/domain-links.proto new file mode 100644 index 00000000..2a31bbf7 --- /dev/null +++ b/code/functions/domain-links/api/src/main/protobuf/domain-links.proto @@ -0,0 +1,29 @@ +syntax="proto3"; +package nu.marginalia.api.domainlinks; + +option java_package="nu.marginalia.api.domainlink"; +option java_multiple_files=true; + +service DomainLinksApi { + rpc getAllLinks(Empty) returns (stream RpcDomainIdPairs) {} + rpc getLinksFromDomain(RpcDomainId) returns (RpcDomainIdList) {} + rpc getLinksToDomain(RpcDomainId) returns (RpcDomainIdList) {} + rpc countLinksFromDomain(RpcDomainId) returns (RpcDomainIdCount) {} + rpc countLinksToDomain(RpcDomainId) returns (RpcDomainIdCount) {} +} + +message RpcDomainId { + int32 domainId = 1; +} +message RpcDomainIdList { + repeated int32 domainId = 1 [packed=true]; +} +message RpcDomainIdCount { + int32 idCount = 1; +} +message RpcDomainIdPairs { + repeated int32 sourceIds = 1 [packed=true]; + repeated int32 destIds = 2 [packed=true]; +} + +message Empty {} \ No newline at end of file diff --git a/code/functions/domain-links/partition/build.gradle b/code/functions/domain-links/partition/build.gradle new file mode 100644 index 00000000..1a5f9dde --- /dev/null +++ b/code/functions/domain-links/partition/build.gradle @@ -0,0 +1,42 @@ +plugins { + id 'java' + + id 'application' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':code:functions:domain-links:api') + + implementation project(':code:common:config') + implementation project(':code:common:service') + implementation project(':code:common:model') + implementation project(':code:common:linkdb') + implementation project(':code:common:db') + implementation project(':code:common:service-discovery') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.spark + implementation libs.opencsv + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito + + +} diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java b/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java similarity index 88% rename from code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java rename to code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java index b368d289..323ec51d 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/svc/IndexDomainLinksService.java +++ b/code/functions/domain-links/partition/src/main/java/nu/marginalia/functions/domainlinks/PartitionDomainLinksService.java @@ -1,22 +1,23 @@ -package nu.marginalia.index.svc; +package nu.marginalia.functions.domainlinks; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.index.api.*; +import nu.marginalia.api.domainlink.Empty; +import nu.marginalia.api.domainlink.*; import nu.marginalia.linkdb.dlinks.DomainLinkDb; /** GRPC service for interrogating domain links */ -public class IndexDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase { +public class PartitionDomainLinksService extends DomainLinksApiGrpc.DomainLinksApiImplBase { private final DomainLinkDb domainLinkDb; @Inject - public IndexDomainLinksService(DomainLinkDb domainLinkDb) { + public PartitionDomainLinksService(DomainLinkDb domainLinkDb) { this.domainLinkDb = domainLinkDb; } - public void getAllLinks(nu.marginalia.index.api.Empty request, - io.grpc.stub.StreamObserver responseObserver) { + public void getAllLinks(Empty request, + io.grpc.stub.StreamObserver responseObserver) { try (var idsConverter = new AllIdsResponseConverter(responseObserver)) { domainLinkDb.forEach(idsConverter::accept); diff --git a/code/api/assistant-api/build.gradle b/code/functions/math/api/build.gradle similarity index 96% rename from code/api/assistant-api/build.gradle rename to code/functions/math/api/build.gradle index e87165d7..0ef23547 100644 --- a/code/api/assistant-api/build.gradle +++ b/code/functions/math/api/build.gradle @@ -11,6 +11,8 @@ java { } } +jar.archiveBaseName = 'math-api' + sourceSets { main { proto { diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java new file mode 100644 index 00000000..c851aee9 --- /dev/null +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathClient.java @@ -0,0 +1,90 @@ +package nu.marginalia.api.math; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import nu.marginalia.service.client.GrpcChannelPoolFactory; +import nu.marginalia.service.client.GrpcSingleNodeChannelPool; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; +import nu.marginalia.service.id.ServiceId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +import nu.marginalia.api.math.model.*; +import nu.marginalia.api.math.MathProtobufCodec.*; + + +@Singleton +public class MathClient { + private static final Logger logger = LoggerFactory.getLogger(MathClient.class); + + private final GrpcSingleNodeChannelPool channelPool; + private final ExecutorService virtualExecutorService = Executors.newVirtualThreadPerTaskExecutor(); + @Inject + public MathClient(GrpcChannelPoolFactory factory) { + this.channelPool = factory.createSingle( + ServiceKey.forGrpcApi(MathApiGrpc.class, ServicePartition.any()), + MathApiGrpc::newBlockingStub); + + } + + public Future dictionaryLookup(String word) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::dictionaryLookup) + .async(virtualExecutorService) + .run(DictionaryLookup.createRequest(word)) + .thenApply(DictionaryLookup::convertResponse); + } + + @SuppressWarnings("unchecked") + public Future> spellCheck(String word) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck) + .async(virtualExecutorService) + .run(SpellCheck.createRequest(word)) + .thenApply(SpellCheck::convertResponse); + } + + public Map> spellCheck(List words, Duration timeout) throws InterruptedException { + List requests = words.stream().map(SpellCheck::createRequest).toList(); + + var future = channelPool.call(MathApiGrpc.MathApiBlockingStub::spellCheck) + .async(virtualExecutorService) + .runFor(requests); + + try { + var results = future.get(); + Map> map = new HashMap<>(); + for (int i = 0; i < words.size(); i++) { + map.put(words.get(i), SpellCheck.convertResponse(results.get(i))); + } + return map; + } + catch (ExecutionException e) { + throw new RuntimeException(e); + } + } + + public Future unitConversion(String value, String from, String to) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::unitConversion) + .async(virtualExecutorService) + .run(UnitConversion.createRequest(from, to, value)) + .thenApply(UnitConversion::convertResponse); + } + + public Future evalMath(String expression) { + return channelPool.call(MathApiGrpc.MathApiBlockingStub::evalMath) + .async(virtualExecutorService) + .run(EvalMath.createRequest(expression)) + .thenApply(EvalMath::convertResponse); + } + + public boolean isAccepting() { + return channelPool.hasChannel(); + } +} diff --git a/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java new file mode 100644 index 00000000..2b865b21 --- /dev/null +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/MathProtobufCodec.java @@ -0,0 +1,66 @@ +package nu.marginalia.api.math; + +import nu.marginalia.api.math.model.DictionaryEntry; +import nu.marginalia.api.math.model.DictionaryResponse; + +import java.util.List; + +public class MathProtobufCodec { + + public static class DictionaryLookup { + public static RpcDictionaryLookupRequest createRequest(String word) { + return RpcDictionaryLookupRequest.newBuilder() + .setWord(word) + .build(); + } + public static DictionaryResponse convertResponse(RpcDictionaryLookupResponse rsp) { + return new DictionaryResponse( + rsp.getWord(), + rsp.getEntriesList().stream().map(DictionaryLookup::convertResponseEntry).toList() + ); + } + + private static DictionaryEntry convertResponseEntry(RpcDictionaryEntry e) { + return new DictionaryEntry(e.getType(), e.getWord(), e.getDefinition()); + } + } + + public static class SpellCheck { + public static RpcSpellCheckRequest createRequest(String text) { + return RpcSpellCheckRequest.newBuilder() + .setText(text) + .build(); + } + + public static List convertResponse(RpcSpellCheckResponse rsp) { + return rsp.getSuggestionsList(); + } + } + + public static class UnitConversion { + public static RpcUnitConversionRequest createRequest(String from, String to, String unit) { + return RpcUnitConversionRequest.newBuilder() + .setFrom(from) + .setTo(to) + .setUnit(unit) + .build(); + } + + public static String convertResponse(RpcUnitConversionResponse rsp) { + return rsp.getResult(); + } + } + + public static class EvalMath { + public static RpcEvalMathRequest createRequest(String expression) { + return RpcEvalMathRequest.newBuilder() + .setExpression(expression) + .build(); + } + + public static String convertResponse(RpcEvalMathResponse rsp) { + return rsp.getResult(); + } + } + +} diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryEntry.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java similarity index 84% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryEntry.java rename to code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java index c40ea97f..084e2e47 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryEntry.java +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryEntry.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.math.model; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryResponse.java b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java similarity index 86% rename from code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryResponse.java rename to code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java index 03fbd2e6..0668b6b2 100644 --- a/code/api/assistant-api/src/main/java/nu/marginalia/assistant/client/model/DictionaryResponse.java +++ b/code/functions/math/api/src/main/java/nu/marginalia/api/math/model/DictionaryResponse.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.client.model; +package nu.marginalia.api.math.model; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/code/functions/math/api/src/main/protobuf/math-api.proto b/code/functions/math/api/src/main/protobuf/math-api.proto new file mode 100644 index 00000000..b29732ee --- /dev/null +++ b/code/functions/math/api/src/main/protobuf/math-api.proto @@ -0,0 +1,57 @@ +syntax="proto3"; +package nu.marginalia.api.math; + +option java_package="nu.marginalia.api.math"; +option java_multiple_files=true; + +service MathApi { + /** Looks up a word in the dictionary. */ + rpc dictionaryLookup(RpcDictionaryLookupRequest) returns (RpcDictionaryLookupResponse) {} + /** Checks the spelling of a text. */ + rpc spellCheck(RpcSpellCheckRequest) returns (RpcSpellCheckResponse) {} + /** Converts a unit from one to another. */ + rpc unitConversion(RpcUnitConversionRequest) returns (RpcUnitConversionResponse) {} + /** Evaluates a mathematical expression. */ + rpc evalMath(RpcEvalMathRequest) returns (RpcEvalMathResponse) {} +} + +message RpcDictionaryLookupRequest { + string word = 1; +} + +message RpcDictionaryLookupResponse { + string word = 1; + repeated RpcDictionaryEntry entries = 2; +} + +message RpcDictionaryEntry { + string type = 1; + string word = 2; + string definition = 3; +} + +message RpcSpellCheckRequest { + string text = 1; +} + +message RpcSpellCheckResponse { + repeated string suggestions = 1; +} + +message RpcUnitConversionRequest { + string unit = 1; + string from = 2; + string to = 3; +} + +message RpcUnitConversionResponse { + string result = 1; +} + +message RpcEvalMathRequest { + string expression = 1; +} + +message RpcEvalMathResponse { + string result = 1; +} diff --git a/code/functions/math/build.gradle b/code/functions/math/build.gradle new file mode 100644 index 00000000..4ffbc11d --- /dev/null +++ b/code/functions/math/build.gradle @@ -0,0 +1,32 @@ +plugins { + id 'java' + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + +dependencies { + implementation project(':third-party:symspell') + implementation project(':code:functions:math:api') + + implementation libs.bundles.slf4j + + implementation libs.prometheus + implementation libs.bundles.grpc + implementation libs.notnull + implementation libs.guice + implementation libs.spark + implementation libs.opencsv + implementation libs.trove + implementation libs.fastutil + implementation libs.bundles.gson + implementation libs.bundles.mariadb + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito +} diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java similarity index 53% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java index 7cf82cd1..66c0ffe5 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantGrpcService.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/MathGrpcService.java @@ -1,34 +1,28 @@ -package nu.marginalia.assistant; +package nu.marginalia.functions.math; import com.google.inject.Inject; import io.grpc.stub.StreamObserver; -import nu.marginalia.assistant.api.*; -import nu.marginalia.assistant.dict.DictionaryService; -import nu.marginalia.assistant.dict.SpellChecker; -import nu.marginalia.assistant.domains.DomainInformationService; -import nu.marginalia.assistant.domains.SimilarDomainsService; -import nu.marginalia.assistant.eval.MathParser; -import nu.marginalia.assistant.eval.Units; +import nu.marginalia.api.math.*; +import nu.marginalia.functions.math.dict.DictionaryService; +import nu.marginalia.functions.math.dict.SpellChecker; +import nu.marginalia.functions.math.eval.MathParser; +import nu.marginalia.functions.math.eval.Units; -public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase { +public class MathGrpcService extends MathApiGrpc.MathApiImplBase { private final DictionaryService dictionaryService; private final SpellChecker spellChecker; private final Units units; private final MathParser mathParser; - private final DomainInformationService domainInformationService; - private final SimilarDomainsService similarDomainsService; + @Inject - public AssistantGrpcService(DictionaryService dictionaryService, - SpellChecker spellChecker, Units units, MathParser mathParser, DomainInformationService domainInformationService, SimilarDomainsService similarDomainsService) + public MathGrpcService(DictionaryService dictionaryService, SpellChecker spellChecker, Units units, MathParser mathParser) { this.dictionaryService = dictionaryService; this.spellChecker = spellChecker; this.units = units; this.mathParser = mathParser; - this.domainInformationService = domainInformationService; - this.similarDomainsService = similarDomainsService; } @Override @@ -95,37 +89,4 @@ public class AssistantGrpcService extends AssistantApiGrpc.AssistantApiImplBase responseObserver.onCompleted(); } - @Override - public void getDomainInfo(RpcDomainId request, StreamObserver responseObserver) { - var ret = domainInformationService.domainInfo(request.getDomainId()); - - ret.ifPresent(responseObserver::onNext); - - responseObserver.onCompleted(); - } - - @Override - public void getSimilarDomains(RpcDomainLinksRequest request, - StreamObserver responseObserver) { - var ret = similarDomainsService.getSimilarDomains(request.getDomainId(), request.getCount()); - - var responseBuilder = RpcSimilarDomains - .newBuilder() - .addAllDomains(ret); - - responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void getLinkingDomains(RpcDomainLinksRequest request, StreamObserver responseObserver) { - var ret = similarDomainsService.getLinkingDomains(request.getDomainId(), request.getCount()); - - var responseBuilder = RpcSimilarDomains - .newBuilder() - .addAllDomains(ret); - - responseObserver.onNext(responseBuilder.build()); - responseObserver.onCompleted(); - } } diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/DictionaryService.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java similarity index 89% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/DictionaryService.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java index 40686f74..630e9035 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/DictionaryService.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/DictionaryService.java @@ -1,10 +1,10 @@ -package nu.marginalia.assistant.dict; +package nu.marginalia.functions.math.dict; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; -import nu.marginalia.assistant.client.model.DictionaryEntry; -import nu.marginalia.assistant.client.model.DictionaryResponse; +import nu.marginalia.api.math.model.DictionaryEntry; +import nu.marginalia.api.math.model.DictionaryResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/SpellChecker.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java similarity index 91% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/SpellChecker.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java index d1710122..988a533d 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/dict/SpellChecker.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/dict/SpellChecker.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.dict; +package nu.marginalia.functions.math.dict; import com.google.inject.Singleton; import symspell.SymSpell; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/MathParser.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java similarity index 99% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/MathParser.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java index 1ec4bb0a..4e37f83d 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/MathParser.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/MathParser.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; import lombok.AllArgsConstructor; import lombok.SneakyThrows; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Unit.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java similarity index 86% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Unit.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java index cdc352c8..2216623b 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Unit.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Unit.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; public class Unit { diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Units.java b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java similarity index 98% rename from code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Units.java rename to code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java index 4cbb141a..bc1cba7b 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/eval/Units.java +++ b/code/functions/math/src/main/java/nu/marginalia/functions/math/eval/Units.java @@ -1,4 +1,4 @@ -package nu.marginalia.assistant.eval; +package nu.marginalia.functions.math.eval; import com.opencsv.CSVReader; import lombok.SneakyThrows; diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java index 6e3dd46c..a3ddc17b 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java @@ -314,6 +314,10 @@ public class MqPersistence { */ public Collection pollInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException { + if (dataSource.isClosed()) { + return Collections.emptyList(); + } + // Mark new messages as claimed int expected = markInboxMessages(inboxName, instanceUUID, tick, n); if (expected == 0) { @@ -366,6 +370,10 @@ public class MqPersistence { */ public Collection pollReplyInbox(String inboxName, String instanceUUID, long tick, int n) throws SQLException { + if (dataSource.isClosed()) { + return Collections.emptyList(); + } + // Mark new messages as claimed int expected = markInboxMessages(inboxName, instanceUUID, tick, n); if (expected == 0) { diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java index 779e2573..27ad832c 100644 --- a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java +++ b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java @@ -76,6 +76,7 @@ public class IndexConstructorMain extends ProcessMainClass { // Grace period so we don't rug pull the logger or jdbc TimeUnit.SECONDS.sleep(5); + System.exit(0); } diff --git a/code/processes/website-adjacencies-calculator/build.gradle b/code/processes/website-adjacencies-calculator/build.gradle index a007c105..d262934e 100644 --- a/code/processes/website-adjacencies-calculator/build.gradle +++ b/code/processes/website-adjacencies-calculator/build.gradle @@ -23,7 +23,7 @@ dependencies { implementation project(':code:common:process') implementation project(':code:common:service-discovery') implementation project(':code:common:service') - implementation project(':code:api:query-api') + implementation project(':code:functions:domain-links:api') implementation libs.bundles.slf4j diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java index 61c2ceee..d0886b9f 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/AdjacenciesData.java @@ -4,7 +4,7 @@ import gnu.trove.list.TIntList; import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.set.hash.TIntHashSet; -import nu.marginalia.query.client.QueryClient; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,14 +35,15 @@ public class AdjacenciesData { return ret; } - public AdjacenciesData(QueryClient queryClient, + public AdjacenciesData(AggregateDomainLinksClient linksClient, DomainAliases aliases) { logger.info("Loading adjacency data"); Map tmpMapDtoS = new HashMap<>(100_000); int count = 0; - var allLinks = queryClient.getAllDomainLinks(); + var allLinks = linksClient.getAllDomainLinks(); + for (var iter = allLinks.iterator();;count++) { if (!iter.advance()) { break; diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java index eef58571..cc720d5e 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java @@ -4,18 +4,20 @@ import com.google.inject.Guice; import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.ProcessConfiguration; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.model.EdgeDomain; import nu.marginalia.process.control.ProcessHeartbeat; import nu.marginalia.process.control.ProcessHeartbeatImpl; -import nu.marginalia.query.client.QueryClient; import nu.marginalia.service.MainClass; +import nu.marginalia.service.ProcessMainClass; import nu.marginalia.service.ServiceDiscoveryModule; import nu.marginalia.service.module.DatabaseModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.SQLException; +import java.time.Duration; import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; @@ -24,18 +26,18 @@ import java.util.stream.IntStream; import static nu.marginalia.adjacencies.SparseBitVector.*; -public class WebsiteAdjacenciesCalculator extends MainClass { +public class WebsiteAdjacenciesCalculator extends ProcessMainClass { private final HikariDataSource dataSource; public AdjacenciesData adjacenciesData; public DomainAliases domainAliases; private static final Logger logger = LoggerFactory.getLogger(WebsiteAdjacenciesCalculator.class); float[] weights; - public WebsiteAdjacenciesCalculator(QueryClient queryClient, HikariDataSource dataSource) throws SQLException { + public WebsiteAdjacenciesCalculator(AggregateDomainLinksClient domainLinksClient, HikariDataSource dataSource) throws SQLException { this.dataSource = dataSource; domainAliases = new DomainAliases(dataSource); - adjacenciesData = new AdjacenciesData(queryClient, domainAliases); + adjacenciesData = new AdjacenciesData(domainLinksClient, domainAliases); weights = adjacenciesData.getWeights(); } @@ -146,16 +148,20 @@ public class WebsiteAdjacenciesCalculator extends MainClass { } - public static void main(String[] args) throws SQLException { + public static void main(String[] args) throws SQLException, InterruptedException { var injector = Guice.createInjector( new DatabaseModule(false), new ServiceDiscoveryModule()); var dataSource = injector.getInstance(HikariDataSource.class); - var qc = injector.getInstance(QueryClient.class); + var lc = injector.getInstance(AggregateDomainLinksClient.class); - var main = new WebsiteAdjacenciesCalculator(qc, dataSource); + if (!lc.waitReady(Duration.ofSeconds(30))) { + throw new IllegalStateException("Failed to connect to domain-links"); + } + + var main = new WebsiteAdjacenciesCalculator(lc, dataSource); if (args.length == 1 && "load".equals(args[0])) { var processHeartbeat = new ProcessHeartbeatImpl( diff --git a/code/services-application/api-service/build.gradle b/code/services-application/api-service/build.gradle index a3e402f7..798fb3a9 100644 --- a/code/services-application/api-service/build.gradle +++ b/code/services-application/api-service/build.gradle @@ -2,8 +2,8 @@ plugins { id 'java' id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } java { @@ -19,7 +19,21 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + dependencies { implementation project(':code:common:db') diff --git a/code/services-application/dating-service/build.gradle b/code/services-application/dating-service/build.gradle index a59476c8..b83eff9a 100644 --- a/code/services-application/dating-service/build.gradle +++ b/code/services-application/dating-service/build.gradle @@ -2,8 +2,8 @@ plugins { id 'java' id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { diff --git a/code/services-application/explorer-service/build.gradle b/code/services-application/explorer-service/build.gradle index 596afdf0..75607fae 100644 --- a/code/services-application/explorer-service/build.gradle +++ b/code/services-application/explorer-service/build.gradle @@ -2,8 +2,8 @@ plugins { id 'java' id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { diff --git a/code/services-application/search-service/build.gradle b/code/services-application/search-service/build.gradle index 80936e48..e7d19834 100644 --- a/code/services-application/search-service/build.gradle +++ b/code/services-application/search-service/build.gradle @@ -2,9 +2,25 @@ plugins { id 'java' id 'io.freefair.sass-base' version '8.4' id 'io.freefair.sass-java' version '8.4' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + + id 'com.google.cloud.tools.jib' version '3.4.0' +} + +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } } application { @@ -14,7 +30,6 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" java { toolchain { @@ -26,6 +41,7 @@ sass { sourceMapEmbed = true outputStyle = EXPANDED } + dependencies { implementation project(':code:common:db') implementation project(':code:common:model') @@ -38,7 +54,8 @@ dependencies { implementation project(':code:libraries:braille-block-punch-cards') implementation project(':code:libraries:term-frequency-dict') - implementation project(':code:api:assistant-api') + implementation project(':code:functions:math:api') + implementation project(':code:functions:domain-info:api') implementation project(':code:api:query-api') implementation project(':code:api:index-api') implementation project(':code:common:service-discovery') diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java index 9c38ffef..28b21da8 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java @@ -4,7 +4,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import lombok.SneakyThrows; import nu.marginalia.WebsiteUrl; -import nu.marginalia.assistant.client.AssistantClient; +import nu.marginalia.api.math.MathClient; import nu.marginalia.model.EdgeDomain; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.query.client.QueryClient; @@ -34,7 +34,7 @@ public class SearchOperator { // Marker for filtering out sensitive content from the persistent logs private final Marker queryMarker = MarkerFactory.getMarker("QUERY"); - private final AssistantClient assistantClient; + private final MathClient mathClient; private final DbDomainQueries domainQueries; private final QueryClient queryClient; private final SearchQueryIndexService searchQueryService; @@ -44,7 +44,7 @@ public class SearchOperator { @Inject - public SearchOperator(AssistantClient assistantClient, + public SearchOperator(MathClient mathClient, DbDomainQueries domainQueries, QueryClient queryClient, SearchQueryIndexService searchQueryService, @@ -53,7 +53,7 @@ public class SearchOperator { SearchUnitConversionService searchUnitConversionService) { - this.assistantClient = assistantClient; + this.mathClient = mathClient; this.domainQueries = domainQueries; this.queryClient = queryClient; @@ -162,7 +162,7 @@ public class SearchOperator { @SneakyThrows private void spellCheckTerms(QueryResponse response) { - var suggestions = assistantClient + var suggestions = mathClient .spellCheck(response.searchTermsHuman(), Duration.ofMillis(20)); suggestions.entrySet() diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java index 52b0dfdb..3025497f 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/DefinitionCommand.java @@ -2,11 +2,11 @@ package nu.marginalia.search.command.commands; import com.google.inject.Inject; -import nu.marginalia.assistant.client.AssistantClient; -import nu.marginalia.assistant.client.model.DictionaryResponse; +import nu.marginalia.api.math.MathClient; +import nu.marginalia.api.math.model.DictionaryResponse; +import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.search.command.SearchCommandInterface; import nu.marginalia.search.command.SearchParameters; -import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,18 +23,18 @@ public class DefinitionCommand implements SearchCommandInterface { private final Logger logger = LoggerFactory.getLogger(getClass()); private final MustacheRenderer dictionaryRenderer; - private final AssistantClient assistantClient; + private final MathClient mathClient; private final Predicate queryPatternPredicate = Pattern.compile("^define:[A-Za-z\\s-0-9]+$").asPredicate(); @Inject - public DefinitionCommand(RendererFactory rendererFactory, AssistantClient assistantClient) + public DefinitionCommand(RendererFactory rendererFactory, MathClient mathClient) throws IOException { dictionaryRenderer = rendererFactory.renderer("search/dictionary-results"); - this.assistantClient = assistantClient; + this.mathClient = mathClient; } @Override @@ -57,9 +57,9 @@ public class DefinitionCommand implements SearchCommandInterface { String word = humanQuery.substring(definePrefix.length()).toLowerCase(); try { - return assistantClient + return mathClient .dictionaryLookup(word) - .get(100, TimeUnit.MILLISECONDS); + .get(250, TimeUnit.MILLISECONDS); } catch (Exception e) { logger.error("Failed to lookup definition for word: " + word, e); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java index 76454538..323d744e 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/command/commands/SearchCommand.java @@ -14,17 +14,13 @@ import java.io.IOException; import java.util.Optional; public class SearchCommand implements SearchCommandInterface { - private final DomainBlacklist blacklist; private final SearchOperator searchOperator; private final MustacheRenderer searchResultsRenderer; @Inject - public SearchCommand(DomainBlacklist blacklist, - SearchOperator searchOperator, - RendererFactory rendererFactory - ) throws IOException { - this.blacklist = blacklist; + public SearchCommand(SearchOperator searchOperator, + RendererFactory rendererFactory) throws IOException { this.searchOperator = searchOperator; searchResultsRenderer = rendererFactory.renderer("search/search-results"); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java index 2dcbbe96..6ba3fa49 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchBrowseService.java @@ -1,8 +1,8 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.assistant.client.AssistantClient; -import nu.marginalia.assistant.client.model.SimilarDomain; +import nu.marginalia.api.domains.DomainInfoClient; +import nu.marginalia.api.domains.model.SimilarDomain; import nu.marginalia.browse.DbBrowseDomainsRandom; import nu.marginalia.browse.model.BrowseResult; import nu.marginalia.browse.model.BrowseResultSet; @@ -25,20 +25,20 @@ public class SearchBrowseService { private final DbBrowseDomainsRandom randomDomains; private final DbDomainQueries domainQueries; private final DomainBlacklist blacklist; - private final AssistantClient assistantClient; + private final DomainInfoClient domainInfoClient; private final BrowseResultCleaner browseResultCleaner; @Inject public SearchBrowseService(DbBrowseDomainsRandom randomDomains, DbDomainQueries domainQueries, DomainBlacklist blacklist, - AssistantClient assistantClient, + DomainInfoClient domainInfoClient, BrowseResultCleaner browseResultCleaner) { this.randomDomains = randomDomains; this.domainQueries = domainQueries; this.blacklist = blacklist; - this.assistantClient = assistantClient; + this.domainInfoClient = domainInfoClient; this.browseResultCleaner = browseResultCleaner; } @@ -53,7 +53,7 @@ public class SearchBrowseService { public BrowseResultSet getRelatedEntries(String domainName) throws ExecutionException, InterruptedException, TimeoutException { var domain = domainQueries.getDomainId(new EdgeDomain(domainName)); - var neighbors = assistantClient.similarDomains(domain, 50) + var neighbors = domainInfoClient.similarDomains(domain, 50) .get(100, TimeUnit.MILLISECONDS); neighbors.removeIf(sd -> !sd.screenshot()); @@ -61,7 +61,7 @@ public class SearchBrowseService { // If the results are very few, supplement with the alternative shitty algorithm if (neighbors.size() < 25) { Set allNeighbors = new HashSet<>(neighbors); - allNeighbors.addAll(assistantClient + allNeighbors.addAll(domainInfoClient .linkedDomains(domain, 50) .get(100, TimeUnit.MILLISECONDS) ); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java index 7f228d7f..7cb5c809 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchSiteInfoService.java @@ -1,8 +1,9 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; -import nu.marginalia.assistant.client.AssistantClient; -import nu.marginalia.assistant.client.model.SimilarDomain; +import nu.marginalia.api.domains.DomainInfoClient; +import nu.marginalia.api.domains.model.DomainInformation; +import nu.marginalia.api.domains.model.SimilarDomain; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.feedlot.model.FeedItems; import nu.marginalia.model.EdgeDomain; @@ -10,7 +11,6 @@ import nu.marginalia.renderer.MustacheRenderer; import nu.marginalia.renderer.RendererFactory; import nu.marginalia.screenshot.ScreenshotService; import nu.marginalia.search.SearchOperator; -import nu.marginalia.assistant.client.model.DomainInformation; import nu.marginalia.feedlot.FeedlotClient; import nu.marginalia.search.model.UrlDetails; import nu.marginalia.search.svc.SearchFlagSiteService.FlagSiteFormData; @@ -32,7 +32,7 @@ public class SearchSiteInfoService { private static final Logger logger = LoggerFactory.getLogger(SearchSiteInfoService.class); private final SearchOperator searchOperator; - private final AssistantClient assistantClient; + private final DomainInfoClient domainInfoClient; private final SearchFlagSiteService flagSiteService; private final DbDomainQueries domainQueries; private final MustacheRenderer renderer; @@ -41,7 +41,7 @@ public class SearchSiteInfoService { @Inject public SearchSiteInfoService(SearchOperator searchOperator, - AssistantClient assistantClient, + DomainInfoClient domainInfoClient, RendererFactory rendererFactory, SearchFlagSiteService flagSiteService, DbDomainQueries domainQueries, @@ -49,7 +49,7 @@ public class SearchSiteInfoService { ScreenshotService screenshotService) throws IOException { this.searchOperator = searchOperator; - this.assistantClient = assistantClient; + this.domainInfoClient = domainInfoClient; this.flagSiteService = flagSiteService; this.domainQueries = domainQueries; @@ -137,15 +137,20 @@ public class SearchSiteInfoService { boolean hasScreenshot = screenshotService.hasScreenshot(domainId); var feedItemsFuture = feedlotClient.getFeedItems(domainName); - if (domainId < 0 || !assistantClient.isAccepting()) { + if (domainId < 0) { + domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID")); + similarSetFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID")); + linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID")); + } + else if (!domainInfoClient.isAccepting()) { domainInfoFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); similarSetFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); linkingDomainsFuture = CompletableFuture.failedFuture(new Exception("Assistant Service Unavailable")); } else { - domainInfoFuture = assistantClient.domainInformation(domainId); - similarSetFuture = assistantClient.similarDomains(domainId, 25); - linkingDomainsFuture = assistantClient.linkedDomains(domainId, 25); + domainInfoFuture = domainInfoClient.domainInformation(domainId); + similarSetFuture = domainInfoClient.similarDomains(domainId, 25); + linkingDomainsFuture = domainInfoClient.linkedDomains(domainId, 25); } List sampleResults = searchOperator.doSiteSearch(domainName, 5); @@ -174,7 +179,7 @@ public class SearchSiteInfoService { private T waitForFuture(Future future, Supplier fallback) { try { - return future.get(50, TimeUnit.MILLISECONDS); + return future.get(250, TimeUnit.MILLISECONDS); } catch (Exception e) { logger.info("Failed to get domain data: {}", e.getMessage()); return fallback.get(); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java index d581c674..1727878e 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchUnitConversionService.java @@ -1,6 +1,6 @@ package nu.marginalia.search.svc; -import nu.marginalia.assistant.client.AssistantClient; +import nu.marginalia.api.math.MathClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,11 +21,11 @@ public class SearchUnitConversionService { private final Pattern conversionPattern = Pattern.compile("((\\d+|\\s+|[.()\\-^+%*/]|log[^a-z]|log2[^a-z]|sqrt[^a-z]|log10|cos[^a-z]|sin[^a-z]|tan[^a-z]|log2|pi[^a-z]|e[^a-z]|2pi[^a-z])+)\\s*([a-zA-Z][a-zA-Z^.0-9]*\\s?[a-zA-Z^.0-9]*)\\s+in\\s+([a-zA-Z^.0-9]+\\s?[a-zA-Z^.0-9]*)"); private final Predicate evalPredicate = Pattern.compile("(\\d+|\\s+|[.()\\-^+%*/]|log|log2|sqrt|log10|cos|sin|tan|pi|e|2pi)+").asMatchPredicate(); - private final AssistantClient assistantClient; + private final MathClient mathClient; @Inject - public SearchUnitConversionService(AssistantClient assistantClient) { - this.assistantClient = assistantClient; + public SearchUnitConversionService(MathClient mathClient) { + this.mathClient = mathClient; } public Optional tryConversion(String query) { @@ -40,9 +40,9 @@ public class SearchUnitConversionService { logger.info("{} -> '{}' '{}' '{}'", query, value, from, to); try { - var resultFuture = assistantClient.unitConversion(value, from, to); + var resultFuture = mathClient.unitConversion(value, from, to); return Optional.of( - resultFuture.get(100, TimeUnit.MILLISECONDS) + resultFuture.get(250, TimeUnit.MILLISECONDS) ); } catch (ExecutionException e) { logger.error("Error in unit conversion", e); @@ -68,6 +68,6 @@ public class SearchUnitConversionService { logger.info("eval({})", expr); - return assistantClient.evalMath(expr); + return mathClient.evalMath(expr); } } diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index b9f7625a..24a11e33 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -3,7 +3,7 @@ plugins { id 'application' id 'jvm-test-suite' - id 'com.palantir.docker' version '0.35.0' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { @@ -23,7 +38,12 @@ java { dependencies { implementation project(':third-party:symspell') - implementation project(':code:api:assistant-api') + + implementation project(':code:functions:math') + implementation project(':code:functions:math:api') + implementation project(':code:functions:domain-info') + implementation project(':code:functions:domain-info:api') + implementation project(':code:api:query-api') implementation project(':code:common:config') implementation project(':code:common:service') diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java index 51240c7a..741261b8 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantService.java @@ -4,8 +4,11 @@ import com.google.gson.Gson; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.assistant.suggest.Suggestions; +import nu.marginalia.functions.domains.DomainInfoGrpcService; +import nu.marginalia.functions.math.MathGrpcService; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.screenshot.ScreenshotService; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,10 +27,13 @@ public class AssistantService extends Service { @Inject public AssistantService(BaseServiceParams params, ScreenshotService screenshotService, - AssistantGrpcService assistantGrpcService, + DomainInfoGrpcService domainInfoGrpcService, + MathGrpcService mathGrpcService, Suggestions suggestions) { - super(params, List.of(assistantGrpcService)); + super(params, + ServicePartition.any(), + List.of(domainInfoGrpcService, mathGrpcService)); this.suggestions = suggestions; diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java index 7adf7921..2e7b28fb 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/suggest/Suggestions.java @@ -2,9 +2,9 @@ package nu.marginalia.assistant.suggest; import com.google.inject.Inject; import com.google.inject.name.Named; +import nu.marginalia.functions.math.dict.SpellChecker; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.model.crawl.HtmlFeature; -import nu.marginalia.assistant.dict.SpellChecker; import org.apache.commons.collections4.trie.PatriciaTrie; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index f55610b7..af7e553f 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -1,9 +1,8 @@ plugins { id 'java' - id 'application' - id 'com.palantir.docker' version '0.35.0' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } java { @@ -19,7 +18,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + dependencies { implementation libs.bundles.gson diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index 264174a5..8eb44279 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -60,6 +60,7 @@ public class ControlService extends Service { ) throws IOException { super(params); + this.monitors = monitors; this.heartbeatService = heartbeatService; this.eventLogService = eventLogService; diff --git a/code/services-core/executor-service/build.gradle b/code/services-core/executor-service/build.gradle index 5ee7ef74..a4c0e973 100644 --- a/code/services-core/executor-service/build.gradle +++ b/code/services-core/executor-service/build.gradle @@ -1,9 +1,9 @@ plugins { id 'java' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,24 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service-with-dist.gradle" +clean { + delete fileTree('build/dist-extra') +} + +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} java { toolchain { @@ -22,6 +39,15 @@ java { } dependencies { + // These look weird but they're needed to be able to spawn the processes + // from the executor service + + implementation project(':code:processes:website-adjacencies-calculator') + implementation project(':code:processes:crawling-process') + implementation project(':code:processes:loading-process') + implementation project(':code:processes:converting-process') + implementation project(':code:processes:index-constructor-process') + implementation project(':code:common:config') implementation project(':code:common:model') implementation project(':code:common:process') @@ -35,6 +61,8 @@ dependencies { implementation project(':code:libraries:message-queue') + implementation project(':code:functions:domain-links:api') + implementation project(':code:process-models:crawl-spec') implementation project(':code:process-models:crawling-model') implementation project(':code:features-crawl:link-parser') diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java index 537a0969..fa75f016 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ExportDataActor.java @@ -6,6 +6,7 @@ import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.actor.prototype.RecordActorPrototype; import nu.marginalia.actor.state.ActorStep; +import nu.marginalia.api.indexdomainlinks.AggregateDomainLinksClient; import nu.marginalia.query.client.QueryClient; import nu.marginalia.storage.FileStorageService; import nu.marginalia.storage.model.FileStorageId; @@ -32,7 +33,7 @@ public class ExportDataActor extends RecordActorPrototype { private final FileStorageService storageService; private final HikariDataSource dataSource; private final Logger logger = LoggerFactory.getLogger(getClass()); - private final QueryClient queryClient; + private final AggregateDomainLinksClient domainLinksClient; public record Export() implements ActorStep {} public record ExportBlacklist(FileStorageId fid) implements ActorStep {} @@ -114,7 +115,7 @@ public class ExportDataActor extends RecordActorPrototype { var tmpFile = Files.createTempFile(storage.asPath(), "export", ".csv.gz", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--"))); - var allLinks = queryClient.getAllDomainLinks(); + var allLinks = domainLinksClient.getAllDomainLinks(); try (var bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(Files.newOutputStream(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))))) { @@ -154,12 +155,13 @@ public class ExportDataActor extends RecordActorPrototype { @Inject public ExportDataActor(Gson gson, FileStorageService storageService, - HikariDataSource dataSource, QueryClient queryClient) + HikariDataSource dataSource, + AggregateDomainLinksClient domainLinksClient) { super(gson); this.storageService = storageService; this.dataSource = dataSource; - this.queryClient = queryClient; + this.domainLinksClient = domainLinksClient; } } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java index 61619823..0c9f08eb 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorModule.java @@ -2,12 +2,14 @@ package nu.marginalia.executor; import com.google.inject.AbstractModule; import com.google.inject.name.Names; +import nu.marginalia.WmsaHome; import java.nio.file.Path; public class ExecutorModule extends AbstractModule { public void configure() { - String dist = System.getProperty("distPath", System.getProperty("WMSA_HOME", "/var/lib/wmsa") + "/dist/current"); + + String dist = System.getProperty("distPath", WmsaHome.getHomePath().resolve("/dist").toString()); bind(Path.class).annotatedWith(Names.named("distPath")).toInstance(Path.of(dist)); } } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java index 0091ac9f..61769f5b 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java @@ -1,10 +1,10 @@ package nu.marginalia.executor; -import com.google.gson.Gson; import com.google.inject.Inject; import nu.marginalia.actor.ExecutorActor; import nu.marginalia.actor.ExecutorActorControlService; import nu.marginalia.executor.svc.TransferService; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; import nu.marginalia.service.server.mq.MqRequest; @@ -16,10 +16,7 @@ import java.util.List; // Weird name for this one to not have clashes with java.util.concurrent.ExecutorService public class ExecutorSvc extends Service { - private final BaseServiceParams params; - private final Gson gson; private final ExecutorActorControlService actorControlService; - private final TransferService transferService; private static final Logger logger = LoggerFactory.getLogger(ExecutorSvc.class); @@ -28,14 +25,12 @@ public class ExecutorSvc extends Service { public ExecutorSvc(BaseServiceParams params, ExecutorActorControlService actorControlService, ExecutorGrpcService executorGrpcService, - Gson gson, TransferService transferService) { - super(params, List.of(executorGrpcService)); - this.params = params; - this.gson = gson; + super(params, + ServicePartition.partition(params.configuration.node()), + List.of(executorGrpcService)); this.actorControlService = actorControlService; - this.transferService = transferService; Spark.get("/transfer/file/:fid", transferService::transferFile); } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java index 971b2ab9..93f89063 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/process/ProcessService.java @@ -3,6 +3,14 @@ package nu.marginalia.process; import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; +import nu.marginalia.WmsaHome; +import nu.marginalia.adjacencies.WebsiteAdjacenciesCalculator; +import nu.marginalia.converting.ConverterMain; +import nu.marginalia.crawl.CrawlerMain; +import nu.marginalia.index.IndexConstructorMain; +import nu.marginalia.loading.LoaderMain; +import nu.marginalia.service.MainClass; +import nu.marginalia.service.ProcessMainClass; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.server.BaseServiceParams; import org.slf4j.Logger; @@ -43,16 +51,32 @@ public class ProcessService { } public enum ProcessId { - CRAWLER("crawler-process/bin/crawler-process"), - CONVERTER("converter-process/bin/converter-process"), - LOADER("loader-process/bin/loader-process"), - INDEX_CONSTRUCTOR("index-construction-process/bin/index-construction-process"), - ADJACENCIES_CALCULATOR("website-adjacencies-calculator/bin/website-adjacencies-calculator") + CRAWLER(CrawlerMain.class), + CONVERTER(ConverterMain.class), + LOADER(LoaderMain.class), + INDEX_CONSTRUCTOR(IndexConstructorMain.class), + ADJACENCIES_CALCULATOR(WebsiteAdjacenciesCalculator.class) ; - public final String path; - ProcessId(String path) { - this.path = path; + public final String mainClass; + ProcessId(Class mainClass) { + this.mainClass = mainClass.getName(); + } + + List envOpts() { + String variable = switch (this) { + case CRAWLER -> "CRAWLER_PROCESS_OPTS"; + case CONVERTER -> "CONVERTER_PROCESS_OPTS"; + case LOADER -> "LOADER_PROCESS_OPTS"; + case INDEX_CONSTRUCTOR -> "INDEX_CONSTRUCTION_PROCESS_OPTS"; + case ADJACENCIES_CALCULATOR -> "ADJACENCIES_CALCULATOR_PROCESS_OPTS"; + }; + String value = System.getenv(variable); + + if (value == null) + return List.of(); + else + return Arrays.asList(value.split("\\s+")); } } @@ -63,27 +87,27 @@ public class ProcessService { this.distPath = distPath; } - public boolean trigger(ProcessId processId) throws Exception { - return trigger(processId, new String[0]); - } - public boolean trigger(ProcessId processId, String... parameters) throws Exception { - final String processPath = distPath.resolve(processId.path).toString(); + public boolean trigger(ProcessId processId, String... extraArgs) throws Exception { final String[] env = createEnvironmentVariables(); - final String[] args = createCommandArguments(processPath, parameters); + List args = new ArrayList<>(); + String javaHome = System.getProperty("java.home"); + + args.add(STR."\{javaHome}/bin/java"); + args.add("-cp"); + args.add(System.getProperty("java.class.path")); + args.add("--enable-preview"); + args.addAll(processId.envOpts()); + args.add(processId.mainClass); + args.addAll(Arrays.asList(extraArgs)); Process process; - if (!Files.exists(Path.of(processPath))) { - logger.error("Process not found: {}", processPath); - return false; - } - - logger.info("Starting process: {}: {} // {}", processId, Arrays.toString(args), Arrays.toString(env)); + logger.info("Starting process: {} {}", processId, processId.envOpts()); synchronized (processes) { if (processes.containsKey(processId)) return false; - process = Runtime.getRuntime().exec(args, env); + process = Runtime.getRuntime().exec(args.toArray(String[]::new), env); processes.put(processId, process); } @@ -107,13 +131,6 @@ public class ProcessService { } - private String[] createCommandArguments(String processPath, String[] parameters) { - final String[] args = new String[parameters.length + 1]; - args[0] = processPath; - System.arraycopy(parameters, 0, args, 1, parameters.length); - return args; - } - public boolean isRunning(ProcessId processId) { return processes.containsKey(processId); } @@ -131,25 +148,14 @@ public class ProcessService { /** These environment variables are propagated from the parent process to the child process, * along with WMSA_HOME, but it has special logic */ private final List propagatedEnvironmentVariables = List.of( - "JAVA_HOME", "ZOOKEEPER_HOSTS", - "WMSA_SERVICE_NODE", - "CONVERTER_PROCESS_OPTS", - "LOADER_PROCESS_OPTS", - "INDEX_CONSTRUCTION_PROCESS_OPTS", - "CRAWLER_PROCESS_OPTS"); + "WMSA_SERVICE_NODE" + ); private String[] createEnvironmentVariables() { List opts = new ArrayList<>(); - String WMSA_HOME = System.getenv("WMSA_HOME"); - - if (WMSA_HOME == null || WMSA_HOME.isBlank()) { - WMSA_HOME = "/var/lib/wmsa"; - } - - opts.add(env2str("WMSA_HOME", WMSA_HOME)); - opts.add(env2str("JAVA_OPTS", "--enable-preview")); // + opts.add(env2str("WMSA_HOME", WmsaHome.getHomePath().toString())); for (String envKey : propagatedEnvironmentVariables) { String envValue = System.getenv(envKey); diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 160773ea..cfeeeb57 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -1,9 +1,9 @@ plugins { id 'java' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { @@ -25,8 +40,13 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:db') implementation project(':code:common:linkdb') + + implementation project(':code:functions:domain-links:partition') + implementation project(':code:functions:domain-links:api') + implementation project(':code:common:service') implementation project(':code:api:index-api') + implementation project(':code:common:service-discovery') implementation project(':code:libraries:array') diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java index 242c6cef..2dd90f78 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexService.java @@ -4,8 +4,9 @@ import com.google.gson.Gson; import com.google.inject.Inject; import lombok.SneakyThrows; import nu.marginalia.IndexLocations; -import nu.marginalia.index.svc.IndexDomainLinksService; +import nu.marginalia.functions.domainlinks.PartitionDomainLinksService; import nu.marginalia.linkdb.dlinks.DomainLinkDb; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.storage.FileStorageService; import nu.marginalia.index.client.IndexMqEndpoints; import nu.marginalia.index.index.SearchIndex; @@ -51,10 +52,14 @@ public class IndexService extends Service { FileStorageService fileStorageService, DocumentDbReader documentDbReader, DomainLinkDb domainLinkDb, - IndexDomainLinksService indexDomainLinksService, + + PartitionDomainLinksService partitionDomainLinksService, + ServiceEventLog eventLog) { - super(params, List.of(indexQueryService, indexDomainLinksService)); + super(params, + ServicePartition.partition(params.configuration.node()), + List.of(indexQueryService, partitionDomainLinksService)); this.opsService = opsService; this.searchIndex = searchIndex; diff --git a/code/services-core/query-service/build.gradle b/code/services-core/query-service/build.gradle index 4f87c11a..a60cdfec 100644 --- a/code/services-core/query-service/build.gradle +++ b/code/services-core/query-service/build.gradle @@ -1,9 +1,9 @@ plugins { id 'java' - id 'com.palantir.docker' version '0.35.0' id 'application' id 'jvm-test-suite' + id 'com.google.cloud.tools.jib' version '3.4.0' } application { @@ -13,7 +13,22 @@ application { tasks.distZip.enabled = false -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + java { toolchain { @@ -35,6 +50,9 @@ dependencies { implementation project(':code:libraries:language-processing') implementation project(':code:libraries:term-frequency-dict') + implementation project(':code:functions:domain-links:api') + implementation project(':code:functions:domain-links:aggregate') + implementation libs.bundles.slf4j implementation libs.spark diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java deleted file mode 100644 index 89de784c..00000000 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCDomainLinksService.java +++ /dev/null @@ -1,90 +0,0 @@ -package nu.marginalia.query; - -import com.google.inject.Inject; -import io.grpc.stub.StreamObserver; -import nu.marginalia.service.client.GrpcMultiNodeChannelPool; -import nu.marginalia.index.api.IndexDomainLinksApiGrpc; -import nu.marginalia.index.api.RpcDomainIdCount; -import nu.marginalia.index.api.RpcDomainIdList; -import nu.marginalia.index.api.RpcDomainIdPairs; -import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.id.ServiceId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class QueryGRPCDomainLinksService extends IndexDomainLinksApiGrpc.IndexDomainLinksApiImplBase { - private static final Logger logger = LoggerFactory.getLogger(QueryGRPCDomainLinksService.class); - private final GrpcMultiNodeChannelPool channelPool; - - @Inject - public QueryGRPCDomainLinksService(GrpcChannelPoolFactory channelPoolFactory) { - channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexDomainLinksApiGrpc::newBlockingStub); - } - - @Override - public void getAllLinks(nu.marginalia.index.api.Empty request, - StreamObserver responseObserver) { - channelPool.callEachSequential(stub -> stub.getAllLinks(request)) - .forEach( - iter -> iter.forEachRemaining(responseObserver::onNext) - ); - - responseObserver.onCompleted(); - } - - @Override - public void getLinksFromDomain(nu.marginalia.index.api.RpcDomainId request, - StreamObserver responseObserver) { - var rspBuilder = RpcDomainIdList.newBuilder(); - - channelPool.callEachSequential(stub -> stub.getLinksFromDomain(request)) - .map(RpcDomainIdList::getDomainIdList) - .forEach(rspBuilder::addAllDomainId); - - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void getLinksToDomain(nu.marginalia.index.api.RpcDomainId request, - StreamObserver responseObserver) { - var rspBuilder = RpcDomainIdList.newBuilder(); - - channelPool.callEachSequential(stub -> stub.getLinksToDomain(request)) - .map(RpcDomainIdList::getDomainIdList) - .forEach(rspBuilder::addAllDomainId); - - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void countLinksFromDomain(nu.marginalia.index.api.RpcDomainId request, - StreamObserver responseObserver) { - - int sum = channelPool.callEachSequential(stub -> stub.countLinksFromDomain(request)) - .mapToInt(RpcDomainIdCount::getIdCount) - .sum(); - - var rspBuilder = RpcDomainIdCount.newBuilder(); - rspBuilder.setIdCount(sum); - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - - @Override - public void countLinksToDomain(nu.marginalia.index.api.RpcDomainId request, - io.grpc.stub.StreamObserver responseObserver) { - - int sum = channelPool.callEachSequential(stub -> stub.countLinksToDomain(request)) - .mapToInt(RpcDomainIdCount::getIdCount) - .sum(); - - var rspBuilder = RpcDomainIdCount.newBuilder(); - rspBuilder.setIdCount(sum); - responseObserver.onNext(rspBuilder.build()); - responseObserver.onCompleted(); - } - -} diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java index b2bb9391..047a2c4c 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java @@ -10,7 +10,8 @@ import nu.marginalia.index.api.*; import nu.marginalia.model.id.UrlIdCodec; import nu.marginalia.query.svc.QueryFactory; import nu.marginalia.service.client.GrpcChannelPoolFactory; -import nu.marginalia.service.id.ServiceId; +import nu.marginalia.service.discovery.property.ServiceKey; +import nu.marginalia.service.discovery.property.ServicePartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +34,7 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { private final QueryFactory queryFactory; private final DomainBlacklist blacklist; + private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); @Inject public QueryGRPCService(QueryFactory queryFactory, @@ -41,7 +43,9 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { { this.queryFactory = queryFactory; this.blacklist = blacklist; - this.channelPool = channelPoolFactory.createMulti(ServiceId.Index, IndexApiGrpc::newBlockingStub); + this.channelPool = channelPoolFactory.createMulti( + ServiceKey.forGrpcApi(IndexApiGrpc.class, ServicePartition.multi()), + IndexApiGrpc::newBlockingStub); } public void query(nu.marginalia.index.api.RpcQsQuery request, @@ -80,39 +84,22 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { @SneakyThrows List executeQueries(RpcIndexQuery indexRequest, int totalSize) { - return channelPool.invokeAll(stub -> new QueryTask(stub, indexRequest)) - .stream() - .filter(f -> f.state() == Future.State.SUCCESS) - .map(Future::resultNow) - .flatMap(List::stream) - .sorted(comparator) - .limit(totalSize) - .toList(); - } - - private class QueryTask implements Callable> { - private final IndexApiGrpc.IndexApiBlockingStub stub; - private final RpcIndexQuery indexRequest; - - public QueryTask(IndexApiGrpc.IndexApiBlockingStub stub, RpcIndexQuery indexRequest) { - this.stub = stub; - this.indexRequest = indexRequest; - } - - @Override - public List call() { - var rsp = stub.query(indexRequest); - List ret = new ArrayList<>(); - - while (rsp.hasNext()) { - RpcDecoratedResultItem next = rsp.next(); - if (isBlacklisted(next)) - continue; - ret.add(next); + var futures = + channelPool.call(IndexApiGrpc.IndexApiBlockingStub::query) + .async(executor) + .runEach(indexRequest); + List results = new ArrayList<>(); + for (var future : futures) { + try { + future.get().forEachRemaining(results::add); + } + catch (Exception e) { + logger.error("Downstream exception", e); } - - return ret; } + results.sort(comparator); + results.removeIf(this::isBlacklisted); + return results.subList(0, Math.min(totalSize, results.size())); } private boolean isBlacklisted(RpcDecoratedResultItem item) { diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java index 0c21558f..6d63be29 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryService.java @@ -3,6 +3,8 @@ package nu.marginalia.query; import com.google.inject.Inject; import io.prometheus.client.Histogram; import lombok.SneakyThrows; +import nu.marginalia.functions.domainlinks.AggregateDomainLinksService; +import nu.marginalia.service.discovery.property.ServicePartition; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; import spark.Spark; @@ -22,12 +24,13 @@ public class QueryService extends Service { @SneakyThrows @Inject public QueryService(BaseServiceParams params, - QueryGRPCDomainLinksService domainLinksService, + AggregateDomainLinksService domainLinksService, QueryGRPCService queryGRPCService, QueryBasicInterface queryBasicInterface) { super(params, () -> Spark.staticFileLocation("/static/"), + ServicePartition.any(), List.of(queryGRPCService, domainLinksService)); diff --git a/code/tools/screenshot-capture-tool/build.gradle b/code/tools/screenshot-capture-tool/build.gradle index 6c1f7c67..b821f7b7 100644 --- a/code/tools/screenshot-capture-tool/build.gradle +++ b/code/tools/screenshot-capture-tool/build.gradle @@ -3,7 +3,7 @@ plugins { id 'application' id 'jvm-test-suite' - id 'com.palantir.docker' version '0.35.0' + id 'com.google.cloud.tools.jib' version '3.4.0' } java { @@ -17,7 +17,21 @@ application { applicationName = 'screenshot-capture-tool' } -apply from: "$rootProject.projectDir/docker-service.gradle" +jib { + from { + image = image = rootProject.ext.dockerImageBase + } + to { + image = 'marginalia/'+project.name + tags = ['latest'] + } + container { + mainClass = application.mainClass + jvmFlags = ['-Dservice.bind-address=0.0.0.0', '-Dservice.useDockerHostname=TRUE', '-Dsystem.homePath=/wmsa'] + volumes = ['/wmsa/conf', '/wmsa/model', '/wmsa/data', '/var/log/wmsa'] + } +} + tasks.distZip.enabled = false diff --git a/docker-service-with-dist.gradle b/docker-service-with-dist.gradle deleted file mode 100644 index c06bd4bd..00000000 --- a/docker-service-with-dist.gradle +++ /dev/null @@ -1,72 +0,0 @@ -ext { - dockerImage='openjdk:21-slim' -} - -tasks.register('dockerFile') { - buildDir.mkdir() - - var df = new File(buildDir, "Dockerfile") - doLast { - df.text = """# -# I'm auto-generated, please don't make changes to me or commit me to git -# -# The template exists in docker-service.gradle -# -FROM ${dockerImage} - -RUN apt-get update && apt-get install -y curl - -ADD ${application.applicationName}.tar / -ADD crawler-process.tar /dist -ADD loader-process.tar /dist -ADD converter-process.tar /dist -ADD website-adjacencies-calculator.tar /dist -ADD index-construction-process.tar /dist - -RUN mkdir /wmsa - -# This will make the service grab the hostname from the HOSTNAME variable -ENV WMSA_IN_DOCKER true - -ENTRYPOINT WMSA_HOME=/wmsa /${application.applicationName}/bin/${application.applicationName} \${arg0} \${arg1} -""" - } - it.outputs.file(df) -} - -dockerPrepare { - dependsOn tasks.dockerFile - - dependsOn project(':code:processes:website-adjacencies-calculator').distTar - dependsOn project(':code:processes:crawling-process').distTar - dependsOn project(':code:processes:loading-process').distTar - dependsOn project(':code:processes:converting-process').distTar - dependsOn project(':code:processes:index-constructor-process').distTar -} - -dockerfileZip { - dependsOn tasks.dockerFile -} - -docker { - dockerfile = tasks.dockerFile.outputs.files.singleFile - - var registry = project.hasProperty('docker-registry') ? project.property('docker-registry') : 'marginalia' - var tagName = project.hasProperty('docker-tag') ? project.property('docker-tag') : 'latest' - - name = registry+'/'+application.applicationName+':'+tagName - tag 'test', (registry+'/'+application.applicationName+':'+tagName) - - files tasks.distTar.outputs, \ - project(':code:processes:crawling-process').distTar.outputs, \ - project(':code:processes:loading-process').distTar.outputs, \ - project(':code:processes:converting-process').distTar.outputs, \ - project(':code:processes:index-constructor-process').distTar.outputs, \ - project(':code:processes:website-adjacencies-calculator').distTar.outputs - - dependsOn project(':code:processes:crawling-process').distTar - dependsOn project(':code:processes:loading-process').distTar - dependsOn project(':code:processes:converting-process').distTar - dependsOn project(':code:processes:index-constructor-process').distTar - dependsOn project(':code:processes:website-adjacencies-calculator').distTar -} diff --git a/docker-service.gradle b/docker-service.gradle deleted file mode 100644 index 25dd0e17..00000000 --- a/docker-service.gradle +++ /dev/null @@ -1,50 +0,0 @@ -ext { - dockerImage='openjdk:21-slim' -} - -tasks.register('dockerFile') { - buildDir.mkdir() - - var df = new File(buildDir, "Dockerfile") - doLast { - df.text = """# -# I'm auto-generated, please don't make changes to me or commit me to git -# -# The template exists in docker-service.gradle -# -FROM ${dockerImage} - -RUN apt-get update && apt-get install -y curl -ADD ${application.applicationName}.tar / -RUN mkdir /wmsa - -# This will make the service grab the hostname from the HOSTNAME variable -ENV WMSA_IN_DOCKER true - -ENTRYPOINT WMSA_HOME=/wmsa /${application.applicationName}/bin/${application.applicationName} \${arg0} \${arg1} -""" - } - it.outputs.file(df) -} - -dockerPrepare { - dependsOn tasks.dockerFile -} - -dockerfileZip { - dependsOn tasks.dockerFile -} - - -docker { - dockerfile = tasks.dockerFile.outputs.files.singleFile - - var registry = project.hasProperty('docker-registry') ? project.property('docker-registry') : 'marginalia' - var tagName = project.hasProperty('docker-tag') ? project.property('docker-tag') : 'latest' - - name = registry+'/'+application.applicationName+':'+tagName - tag 'test', (registry+'/'+application.applicationName+':'+tagName) - - files tasks.distTar.outputs - dependsOn tasks.distTar -} diff --git a/settings.gradle b/settings.gradle index 79aeace2..49462b94 100644 --- a/settings.gradle +++ b/settings.gradle @@ -11,6 +11,16 @@ include 'code:services-application:api-service' include 'code:services-application:dating-service' include 'code:services-application:explorer-service' +include 'code:functions:math' +include 'code:functions:math:api' + +include 'code:functions:domain-info' +include 'code:functions:domain-info:api' + +include 'code:functions:domain-links:partition' +include 'code:functions:domain-links:aggregate' +include 'code:functions:domain-links:api' + include 'code:libraries:array' include 'code:libraries:geo-ip' include 'code:libraries:btree' @@ -55,7 +65,6 @@ include 'code:features-index:domain-ranking' include 'code:api:query-api' include 'code:api:index-api' -include 'code:api:assistant-api' include 'code:api:process-mqapi' include 'code:api:executor-api' @@ -174,7 +183,7 @@ dependencyResolutionManagement { library('bucket4j','com.github.vladimir-bukhtoyarov','bucket4j-core').version('7.5.0') library('gson','com.google.code.gson','gson').version('2.10.1') - library('gson-type-adapter','com.github.Marcono1234','gson-record-type-adapter-factory').version('0.2.0') + library('gson-type-adapter','com.github.Marcono1234','gson-record-type-adapter-factory').version('0.3.0') library('zstd','com.github.luben','zstd-jni').version('1.5.2-2') library('lz4','org.lz4','lz4-java').version('1.8.0') @@ -221,8 +230,5 @@ dependencyResolutionManagement { bundle('curator', ['curator-framework', 'curator-x-discovery']) } - - } - }