diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java index 4bf19ad1..4d6bae25 100644 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -62,9 +62,9 @@ public class ExecutorClient extends AbstractDynamicClient { // FIXME this shouldn't be done in the executor } - public void sideloadEncyclopedia(Context ctx, int node, Path sourcePath) { + public void sideloadEncyclopedia(Context ctx, int node, Path sourcePath, String baseUrl) { post(ctx, node, - "/sideload/encyclopedia?path="+ URLEncoder.encode(sourcePath.toString(), StandardCharsets.UTF_8), + "/sideload/encyclopedia?path="+ URLEncoder.encode(sourcePath.toString(), StandardCharsets.UTF_8) + "&baseUrl=" + URLEncoder.encode(baseUrl, StandardCharsets.UTF_8), "").blockingSubscribe(); } diff --git a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java index c5241914..fffed79b 100644 --- a/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java +++ b/code/api/process-mqapi/src/main/java/nu/marginalia/mqapi/converting/ConvertRequest.java @@ -3,10 +3,47 @@ package nu.marginalia.mqapi.converting; import lombok.AllArgsConstructor; import nu.marginalia.storage.model.FileStorageId; +import java.nio.file.Path; + @AllArgsConstructor public class ConvertRequest { public final ConvertAction action; public final String inputSource; public final FileStorageId crawlStorage; public final FileStorageId processedDataStorage; + public final String baseUrl; + + public static ConvertRequest forCrawlData(FileStorageId sourceId, FileStorageId destId) { + return new ConvertRequest( + ConvertAction.ConvertCrawlData, + null, + sourceId, + destId, + null + ); + } + + public static ConvertRequest forEncyclopedia(Path sourcePath, String baseUrl, FileStorageId destId) { + return new ConvertRequest(ConvertAction.SideloadEncyclopedia, + sourcePath.toString(), + null, + destId, + baseUrl); + } + + public static ConvertRequest forDirtree(Path sourcePath, FileStorageId destId) { + return new ConvertRequest(ConvertAction.SideloadDirtree, + sourcePath.toString(), + null, + destId, + null); + } + + public static ConvertRequest forStackexchange(Path sourcePath, FileStorageId destId) { + return new ConvertRequest(ConvertAction.SideloadStackexchange, + sourcePath.toString(), + null, + destId, + null); + } } diff --git a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java index c6db522c..bf6eefc6 100644 --- a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java @@ -49,17 +49,18 @@ public class FileStorageService { public FileStorageBase getStorageBase(FileStorageBaseId id) throws SQLException { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT ID, NAME, PATH, TYPE + SELECT ID, NAME, NODE, PATH, TYPE FROM FILE_STORAGE_BASE WHERE ID = ? """)) { stmt.setLong(1, id.id()); try (var rs = stmt.executeQuery()) { if (rs.next()) { return new FileStorageBase( - new FileStorageBaseId(rs.getLong(1)), - FileStorageBaseType.valueOf(rs.getString(4)), - rs.getString(2), - rs.getString(3) + new FileStorageBaseId(rs.getLong("ID")), + FileStorageBaseType.valueOf(rs.getString("TYPE")), + rs.getInt("NODE"), + rs.getString("NAME"), + rs.getString("PATH") ); } } @@ -154,7 +155,7 @@ public class FileStorageService { public FileStorageBase getStorageBase(FileStorageBaseType type, int node) throws SQLException { try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(""" - SELECT ID, NAME, PATH, TYPE + SELECT ID, NAME, NODE, PATH, TYPE FROM FILE_STORAGE_BASE WHERE TYPE = ? AND NODE = ? """)) { stmt.setString(1, type.name()); @@ -162,10 +163,11 @@ public class FileStorageService { try (var rs = stmt.executeQuery()) { if (rs.next()) { return new FileStorageBase( - new FileStorageBaseId(rs.getLong(1)), - FileStorageBaseType.valueOf(rs.getString(4)), - rs.getString(2), - rs.getString(3) + new FileStorageBaseId(rs.getLong("ID")), + FileStorageBaseType.valueOf(rs.getString("TYPE")), + rs.getInt("NODE"), + rs.getString("NAME"), + rs.getString("PATH") ); } } diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java index f6090d56..7994622a 100644 --- a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorage.java @@ -30,6 +30,7 @@ public record FileStorage ( var mockBase = new FileStorageBase( new FileStorageBaseId(-1), baseType, + -1, "OVERRIDE:" + type.name(), "INVALIDINVALIDINVALID" ); @@ -45,6 +46,9 @@ public record FileStorage ( ); } + public int node() { + return base.node(); + } public Path asPath() { return Path.of(path); } diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java index f8d78089..88b21f2f 100644 --- a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java +++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBase.java @@ -12,6 +12,7 @@ import java.nio.file.Path; */ public record FileStorageBase(FileStorageBaseId id, FileStorageBaseType type, + int node, String name, String path ) { diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java index f658e1a1..f144e300 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/outbox/MqOutbox.java @@ -1,5 +1,6 @@ package nu.marginalia.mq.outbox; +import com.google.gson.Gson; import nu.marginalia.mq.MqMessage; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.persistence.MqPersistence; @@ -25,7 +26,7 @@ public class MqOutbox { private final int pollIntervalMs = Integer.getInteger("mq.outbox.poll-interval-ms", 1000); private final int maxPollCount = Integer.getInteger("mq.outbox.max-poll-count", 10); private final Thread pollThread; - + private final Gson gson; private volatile boolean run = true; public MqOutbox(MqPersistence persistence, @@ -35,6 +36,7 @@ public class MqOutbox { int outboxNode, UUID instanceUUID) { this.persistence = persistence; + this.gson = persistence.getGson(); this.inboxName = inboxName + ":" + inboxNode; this.replyInboxName = String.format("%s:%d//%s:%d", outboxName, outboxNode, inboxName, inboxNode); @@ -97,6 +99,13 @@ public class MqOutbox { return waitResponse(id); } + /** Send a message and wait for a response */ + public MqMessage send(Object object) throws Exception { + final long id = sendAsync(object); + + return waitResponse(id); + } + /** Send a message asynchronously, without waiting for a response. *
* Use waitResponse(id) or pollResponse(id) to fetch the response. */ @@ -104,6 +113,15 @@ public class MqOutbox { return persistence.sendNewMessage(inboxName, replyInboxName, null, function, payload, null); } + /** Send a message asynchronously, without waiting for a response. + *
+ * Use waitResponse(id) or pollResponse(id) to fetch the response. */ + public long sendAsync(Object request) throws Exception { + return persistence.sendNewMessage(inboxName, replyInboxName, null, + request.getClass().getSimpleName(), + gson.toJson(request), + null); + } /** Blocks until a response arrives for the given message id (possibly forever) */ public MqMessage waitResponse(long id) throws Exception { synchronized (pendingResponses) { @@ -156,13 +174,27 @@ public class MqOutbox { return Optional.ofNullable(response); } - public long sendNotice(String function, String payload) throws Exception { - return persistence.sendNewMessage(inboxName, null, null, function, payload, null); - } + + public long sendNotice(long relatedId, String function, String payload) throws Exception { return persistence.sendNewMessage(inboxName, null, relatedId, function, payload, null); } + public long sendNotice(String function, String payload) throws Exception { + return persistence.sendNewMessage(inboxName, null, null, function, payload, null); + } + public long sendNotice(long relatedId, Object object) throws Exception { + return persistence.sendNewMessage(inboxName, null, relatedId, + object.getClass().getSimpleName(), + gson.toJson(object), + null); + } + public long sendNotice(Object object) throws Exception { + return persistence.sendNewMessage(inboxName, null, null, + object.getClass().getSimpleName(), + gson.toJson(object), + null); + } public void flagAsBad(long id) throws SQLException { persistence.updateMessageState(id, MqMessageState.ERR); } diff --git a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java index 68fb2f83..12c4c987 100644 --- a/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java +++ b/code/libraries/message-queue/src/main/java/nu/marginalia/mq/persistence/MqPersistence.java @@ -1,6 +1,7 @@ package nu.marginalia.mq.persistence; import com.google.common.collect.Lists; +import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; @@ -21,10 +22,16 @@ import static nu.marginalia.mq.MqMessageState.NEW; @Singleton public class MqPersistence { private final HikariDataSource dataSource; + private final Gson gson; - @Inject public MqPersistence(HikariDataSource dataSource) { this.dataSource = dataSource; + this.gson = null; + } + @Inject + public MqPersistence(HikariDataSource dataSource, Gson gson) { + this.dataSource = dataSource; + this.gson = gson; } /** @@ -484,4 +491,8 @@ public class MqPersistence { return ret; } } + + public Gson getGson() { + return gson; + } } diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java index e4f9e36c..20ded57d 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java @@ -244,7 +244,7 @@ public class ConverterMain { case SideloadEncyclopedia -> { var processData = fileStorageService.getStorage(request.processedDataStorage); - yield new SideloadAction(sideloadSourceFactory.sideloadEncyclopediaMarginaliaNu(Path.of(request.inputSource)), + yield new SideloadAction(sideloadSourceFactory.sideloadEncyclopediaMarginaliaNu(Path.of(request.inputSource), request.baseUrl), processData.asPath(), msg, inbox); } diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java index 76f4de9c..6b5b4f7a 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/SideloadSourceFactory.java @@ -34,8 +34,8 @@ public class SideloadSourceFactory { this.dirtreeSideloaderFactory = dirtreeSideloaderFactory; } - public SideloadSource sideloadEncyclopediaMarginaliaNu(Path pathToDbFile) throws SQLException { - return new EncyclopediaMarginaliaNuSideloader(pathToDbFile, gson, sideloaderProcessing); + public SideloadSource sideloadEncyclopediaMarginaliaNu(Path pathToDbFile, String baseUrl) throws SQLException { + return new EncyclopediaMarginaliaNuSideloader(pathToDbFile, baseUrl, gson, sideloaderProcessing); } public Collection sideloadDirtree(Path pathToYamlFile) throws IOException { diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java index 40a5c11b..631a2ef1 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/sideload/encyclopedia/EncyclopediaMarginaliaNuSideloader.java @@ -8,7 +8,7 @@ import nu.marginalia.converting.model.ProcessedDocument; import nu.marginalia.converting.model.ProcessedDomain; import nu.marginalia.converting.sideload.SideloadSource; import nu.marginalia.converting.sideload.SideloaderProcessing; -import nu.marginalia.model.EdgeDomain; +import nu.marginalia.model.EdgeUrl; import nu.marginalia.model.crawl.DomainIndexingState; import java.io.ByteArrayInputStream; @@ -35,12 +35,15 @@ import java.util.concurrent.atomic.AtomicBoolean; public class EncyclopediaMarginaliaNuSideloader implements SideloadSource, AutoCloseable { private final Connection connection; + private final EdgeUrl baseUrl; private final Gson gson; private final SideloaderProcessing sideloaderProcessing; public EncyclopediaMarginaliaNuSideloader(Path pathToDbFile, + String baseUrl, Gson gson, SideloaderProcessing sideloaderProcessing) throws SQLException { + this.baseUrl = EdgeUrl.parse(baseUrl).orElseThrow(AssertionError::new); this.gson = gson; this.sideloaderProcessing = sideloaderProcessing; String sqliteDbString = "jdbc:sqlite:" + pathToDbFile.toString(); @@ -53,7 +56,7 @@ public class EncyclopediaMarginaliaNuSideloader implements SideloadSource, AutoC public ProcessedDomain getDomain() { var ret = new ProcessedDomain(); - ret.domain = new EdgeDomain("encyclopedia.marginalia.nu"); + ret.domain = baseUrl.getDomain(); ret.ip = "0.0.0.0"; ret.state = DomainIndexingState.ACTIVE; @@ -138,7 +141,7 @@ public class EncyclopediaMarginaliaNuSideloader implements SideloadSource, AutoC } private ProcessedDocument convertDocument(List parts, String title, String url) throws URISyntaxException, DisqualifiedException { - String fullUrl = "https://encyclopedia.marginalia.nu/article/"+url; + String fullUrl = baseUrl.toString() + url; StringBuilder fullHtml = new StringBuilder(); fullHtml.append("").append(title).append(""); diff --git a/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java b/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java index 14f188ee..0a45487e 100644 --- a/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java +++ b/code/processes/loading-process/src/test/java/nu/marginalia/loading/loader/LoaderIndexJournalWriterTest.java @@ -32,7 +32,7 @@ class LoaderIndexJournalWriterTest { tempDir = Files.createTempDirectory(getClass().getSimpleName()); FileStorageService storageService = Mockito.mock(FileStorageService.class); - Mockito.when(storageService.getStorageBase(FileStorageBaseType.CURRENT)).thenReturn(new FileStorageBase(null, null, null, tempDir.toString())); + Mockito.when(storageService.getStorageBase(FileStorageBaseType.CURRENT)).thenReturn(new FileStorageBase(null, null, 1,null, tempDir.toString())); writer = new LoaderIndexJournalWriter(storageService); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java index 460626a0..4ef7fd19 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java @@ -55,12 +55,13 @@ public class ControlNodeActionsService { Spark.halt(404); return "No such file " + sourcePath; } + String baseUrl = request.queryParams("baseUrl"); final int nodeId = Integer.parseInt(request.params("node")); eventLog.logEvent("USER-ACTION", "SIDELOAD ENCYCLOPEDIA " + nodeId); - executorClient.sideloadEncyclopedia(Context.fromRequest(request), nodeId, sourcePath); + executorClient.sideloadEncyclopedia(Context.fromRequest(request), nodeId, sourcePath, baseUrl); return ""; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java index 9c1bfdcd..35bc44f3 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java @@ -331,7 +331,7 @@ public class ControlNodeService { for (var type : FileStorageBaseType.values()) { var base = fileStorageService.getStorageBase(type, nodeId); bases.add(Objects.requireNonNullElseGet(base, - () -> new FileStorageBase(new FileStorageBaseId(-1), type, "MISSING", "MISSING")) + () -> new FileStorageBase(new FileStorageBaseId(-1), type, -1, "MISSING", "MISSING")) ); } diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb index abe55941..e9c394a7 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb @@ -51,7 +51,13 @@
- + + +
+ +
+ +
diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb index 0b51fb1d..177d68e9 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb @@ -89,13 +89,15 @@ {{#each storage}} Type + Node + Path Name - Path {{base.type}} + {{base.node}} + {{base.path}} {{base.name}} - {{base.path}} Created diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java index 6e9c5b48..a05700fe 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/monitor/AbstractProcessSpawnerActor.java @@ -36,6 +36,7 @@ public class AbstractProcessSpawnerActor extends RecordActorPrototype { public record Initial() implements ActorStep {} @Resume(behavior = ActorResumeBehavior.RETRY) public record Monitor(int errorAttempts) implements ActorStep {} + @Resume(behavior = ActorResumeBehavior.RESTART) public record Run(int attempts) implements ActorStep {} @Terminal public record Aborted() implements ActorStep {} diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java index 0ff1b673..6717ac20 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/proc/ProcessLivenessMonitorActor.java @@ -5,7 +5,9 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.actor.prototype.RecordActorPrototype; +import nu.marginalia.actor.state.ActorResumeBehavior; import nu.marginalia.actor.state.ActorStep; +import nu.marginalia.actor.state.Resume; import nu.marginalia.process.ProcessService; import nu.marginalia.service.control.ServiceEventLog; import nu.marginalia.service.module.ServiceConfiguration; @@ -24,6 +26,7 @@ public class ProcessLivenessMonitorActor extends RecordActorPrototype { private final int node; public record Initial() implements ActorStep {} + @Resume(behavior = ActorResumeBehavior.RESTART) public record Monitor() implements ActorStep {} @Override diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java index a5db8769..275f4092 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertActor.java @@ -30,7 +30,7 @@ public class ConvertActor extends RecordActorPrototype { private final Gson gson; public record Convert(FileStorageId fid) implements ActorStep {}; - public record ConvertEncyclopedia(String source) implements ActorStep {}; + public record ConvertEncyclopedia(String source, String baseUrl) implements ActorStep {}; public record ConvertDirtree(String source) implements ActorStep {}; public record ConvertStackexchange(String source) implements ActorStep {}; @Resume(behavior = ActorResumeBehavior.RETRY) @@ -50,15 +50,9 @@ public class ConvertActor extends RecordActorPrototype { storageService.relateFileStorages(toProcess.id(), processedArea.id()); storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); - // Pre-send convert request - var request = new ConvertRequest(ConvertAction.ConvertCrawlData, - null, - fid, - processedArea.id()); - yield new ConvertWait( processedArea.id(), - mqConverterOutbox.sendAsync(ConvertRequest.class.getSimpleName(), gson.toJson(request)) + mqConverterOutbox.sendAsync(ConvertRequest.forCrawlData(fid, processedArea.id())) ); } case ConvertDirtree(String source) -> { @@ -75,18 +69,12 @@ public class ConvertActor extends RecordActorPrototype { storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); - // Pre-send convert request - var request = new ConvertRequest(ConvertAction.SideloadDirtree, - sourcePath.toString(), - null, - processedArea.id()); - yield new ConvertWait( processedArea.id(), - mqConverterOutbox.sendAsync(ConvertRequest.class.getSimpleName(), gson.toJson(request)) + mqConverterOutbox.sendAsync(ConvertRequest.forDirtree(sourcePath, processedArea.id())) ); } - case ConvertEncyclopedia(String source) -> { + case ConvertEncyclopedia(String source, String baseUrl) -> { Path sourcePath = Path.of(source); if (!Files.exists(sourcePath)) @@ -101,16 +89,9 @@ public class ConvertActor extends RecordActorPrototype { storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); - // Pre-send convert request - var request = new ConvertRequest(ConvertAction.SideloadEncyclopedia, - sourcePath.toString(), - null, - processedArea.id()); - - yield new ConvertWait( processedArea.id(), - mqConverterOutbox.sendAsync(ConvertRequest.class.getSimpleName(), gson.toJson(request)) + mqConverterOutbox.sendAsync(ConvertRequest.forEncyclopedia(sourcePath, baseUrl, processedArea.id())) ); } case ConvertStackexchange(String source) -> { @@ -129,14 +110,10 @@ public class ConvertActor extends RecordActorPrototype { storageService.setFileStorageState(processedArea.id(), FileStorageState.NEW); // Pre-send convert request - var request = new ConvertRequest(ConvertAction.SideloadStackexchange, - sourcePath.toString(), - null, - processedArea.id()); yield new ConvertWait( processedArea.id(), - mqConverterOutbox.sendAsync(ConvertRequest.class.getSimpleName(), gson.toJson(request)) + mqConverterOutbox.sendAsync(ConvertRequest.forStackexchange(sourcePath, processedArea.id())) ); } case ConvertWait(FileStorageId destFid, long msgId) -> { diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java index ea193c9a..bbec71d5 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/ConvertAndLoadActor.java @@ -116,14 +116,8 @@ public class ConvertAndLoadActor extends RecordActorPrototype { yield new Convert(fid, processedArea.id()); } - case Convert(FileStorageId crawlId, FileStorageId processedId, long msgId) when msgId < 0 -> { - var request = new ConvertRequest(ConvertAction.ConvertCrawlData, - null, - crawlId, - processedId); - yield new Convert(crawlId, processedId, - mqConverterOutbox.sendAsync(ConvertRequest.class.getSimpleName(), gson.toJson(request))); - } + case Convert(FileStorageId crawlId, FileStorageId processedId, long msgId) when msgId < 0 -> + new Convert(crawlId, processedId, mqConverterOutbox.sendAsync(ConvertRequest.forCrawlData(crawlId, processedId))); case Convert(FileStorageId crawlId, FileStorageId processedId, long msgId) -> { var rsp = processWatcher.waitResponse(mqConverterOutbox, ProcessService.ProcessId.CONVERTER, msgId); @@ -133,8 +127,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype { yield new Load(List.of(processedId)); } case Load(List processedIds, long msgId) when msgId < 0 -> { - var request = new LoadRequest(processedIds); - long id = mqLoaderOutbox.sendAsync(LoadRequest.class.getSimpleName(), gson.toJson(request)); + long id = mqLoaderOutbox.sendAsync(new LoadRequest(processedIds)); yield new Load(processedIds, id); } @@ -201,8 +194,7 @@ public class ConvertAndLoadActor extends RecordActorPrototype { } private long createIndex(IndexName index) throws Exception { - return mqIndexConstructorOutbox.sendAsync(CreateIndexRequest.class.getSimpleName(), - gson.toJson(new CreateIndexRequest(index))); + return mqIndexConstructorOutbox.sendAsync(new CreateIndexRequest(index)); } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlActor.java index 6e0f0a6d..a77a575b 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/CrawlActor.java @@ -54,8 +54,7 @@ public class CrawlActor extends RecordActorPrototype { storageService.relateFileStorages(storage.id(), dataArea.id()); // Send convert request - var request = new CrawlRequest(List.of(fid), dataArea.id()); - long msgId = mqCrawlerOutbox.sendAsync(CrawlRequest.class.getSimpleName(), gson.toJson(request)); + long msgId = mqCrawlerOutbox.sendAsync(new CrawlRequest(List.of(fid), dataArea.id())); yield new Crawl(msgId); } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RecrawlActor.java b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RecrawlActor.java index 5604ec44..9a98497c 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RecrawlActor.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/actor/task/RecrawlActor.java @@ -46,8 +46,7 @@ public class RecrawlActor extends RecordActorPrototype { refreshService.synchronizeDomainList(); - var request = new CrawlRequest(null, fid); - long id = mqCrawlerOutbox.sendAsync(CrawlRequest.class.getSimpleName(), gson.toJson(request)); + long id = mqCrawlerOutbox.sendAsync(new CrawlRequest(null, fid)); yield new Crawl(id); } diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java index 6991eb90..a74e70ee 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java @@ -21,7 +21,11 @@ public class SideloadService { } public Object sideloadEncyclopedia(Request request, Response response) throws Exception { - actorControlService.startFrom(ExecutorActor.CONVERT, new ConvertActor.ConvertEncyclopedia(request.queryParams("path"))); + actorControlService.startFrom(ExecutorActor.CONVERT, + new ConvertActor.ConvertEncyclopedia( + request.queryParams("path"), + request.queryParams("baseUrl") + )); return ""; } diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java index b6abed22..aa9fdaf5 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryGRPCService.java @@ -86,6 +86,9 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { private final ExecutorService es = Executors.newVirtualThreadPerTaskExecutor(); + private static final Comparator comparator = + Comparator.comparing(RpcDecoratedResultItem::getRankingScore); + private List executeQueries(RpcIndexQuery indexRequest, int totalSize) throws InterruptedException { List>> tasks = createTasks(indexRequest); @@ -119,8 +122,6 @@ public class QueryGRPCService extends QueryApiGrpc.QueryApiImplBase { return tasks; } - private static final Comparator comparator = - Comparator.comparing(RpcDecoratedResultItem::getRankingScore); private boolean isBlacklisted(RpcDecoratedResultItem item) { return blacklist.isBlacklisted(UrlIdCodec.getDomainId(item.getRawItem().getCombinedId()));