diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java index 4d6bae25..2f6abfd5 100644 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -86,6 +86,13 @@ public class ExecutorClient extends AbstractDynamicClient { .blockingSubscribe(); } + public void exportAtags(Context ctx, int node, String fid) { + post(ctx, node, "/export/atags?fid="+fid, "").blockingSubscribe(); + } + public void exportData(Context ctx, int node) { + post(ctx, node, "/export/data", "").blockingSubscribe(); + } + public void restoreBackup(Context context, int node, String fid) { post(context, node, "/backup/" + fid + "/restore", "").blockingSubscribe(); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithActions.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithActions.java index 4731d4bb..b57bd5c0 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithActions.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/model/FileStorageWithActions.java @@ -12,6 +12,9 @@ public record FileStorageWithActions(FileStorage storage) { public boolean isCrawlable() { return storage.type() == FileStorageType.CRAWL_SPEC; } + public boolean isAtagsExportable() { + return storage.type() == FileStorageType.CRAWL_DATA; + } public boolean isRecrawlable() { return storage.type() == FileStorageType.CRAWL_DATA; } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java index 29059b84..fcc396d3 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java @@ -111,11 +111,26 @@ public class ControlNodeService { Spark.post("/public/nodes/:id/storage/backup-restore/:fid", this::triggerRestoreBackup, redirectControl.renderRedirectAcknowledgement("Restoring", "..") ); - + Spark.post("/public/nodes/:id/actions/export-data", this::exportData, + redirectControl.renderRedirectAcknowledgement("Exporting", "../storage/exports") + ); + Spark.post("/public/nodes/:id/storage/:fid/export-atags", this::exportAtags, + redirectControl.renderRedirectAcknowledgement("Exporting", "../../storage/exports") + ); Spark.post("/public/nodes/:id/fsms/:fsm/start", this::startFsm); Spark.post("/public/nodes/:id/fsms/:fsm/stop", this::stopFsm); } + private Object exportData(Request req, Response rsp) { + executorClient.exportData(Context.fromRequest(req), Integer.parseInt(req.params("id"))); + return ""; + } + + private Object exportAtags(Request req, Response rsp) { + executorClient.exportAtags(Context.fromRequest(req), Integer.parseInt(req.params("id")), req.params("fid")); + return ""; + } + public Object startFsm(Request req, Response rsp) throws Exception { executorClient.startFsm(Context.fromRequest(req), Integer.parseInt(req.params("id")), req.params("fsm").toUpperCase()); diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb index 2ba96bc0..f83b8034 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb @@ -288,6 +288,34 @@ + + +
+

+ +

+ +
+ This will export database data: Domains, blacklist and domain links. + +
+
+
+
+ +
+
+
+
+
+
diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb index b1ed3377..d0f34bc6 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb @@ -67,6 +67,14 @@ {{/if}} + {{#if isAtagsExportable}} +
+ + Export anchor tags from this crawl data + + +
+ {{/if}} {{#if isRestorable}}
diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java index 56c5c14f..7fe08e5d 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java @@ -8,10 +8,7 @@ import nu.marginalia.actor.ExecutorActorControlService; import nu.marginalia.actor.state.ActorStateInstance; import nu.marginalia.executor.model.ActorRunState; import nu.marginalia.executor.model.ActorRunStates; -import nu.marginalia.executor.svc.BackupService; -import nu.marginalia.executor.svc.ProcessingService; -import nu.marginalia.executor.svc.SideloadService; -import nu.marginalia.executor.svc.TransferService; +import nu.marginalia.executor.svc.*; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; import nu.marginalia.service.server.mq.MqNotification; @@ -44,6 +41,7 @@ public class ExecutorSvc extends Service { ProcessingService processingService, SideloadService sideloadService, BackupService backupService, + ExportService exportService, FileStorageService fileStorageService, Gson gson, TransferService transferService, @@ -72,6 +70,9 @@ public class ExecutorSvc extends Service { Spark.post("/sideload/stackexchange", sideloadService::sideloadStackexchange); Spark.post("/sideload/encyclopedia", sideloadService::sideloadEncyclopedia); + Spark.post("/export/atags", exportService::exportAtags); + Spark.post("/export/data", exportService::exportData); + Spark.post("/backup/:fid/restore", backupService::restore); Spark.get("/storage/:fid", transferService::listFiles, gson::toJson); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ExportService.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ExportService.java new file mode 100644 index 00000000..6baaf5de --- /dev/null +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/ExportService.java @@ -0,0 +1,31 @@ +package nu.marginalia.executor.svc; + +import com.google.inject.Inject; +import nu.marginalia.actor.ExecutorActor; +import nu.marginalia.actor.ExecutorActorControlService; +import nu.marginalia.actor.task.ConvertActor; +import nu.marginalia.actor.task.ExportAtagsActor; +import nu.marginalia.actor.task.ExportDataActor; +import nu.marginalia.storage.model.FileStorageId; +import spark.Request; +import spark.Response; + +public class ExportService { + private final ExecutorActorControlService actorControlService; + + @Inject + public ExportService(ExecutorActorControlService actorControlService) { + this.actorControlService = actorControlService; + } + + public Object exportData(Request request, Response response) throws Exception { + actorControlService.startFrom(ExecutorActor.EXPORT_DATA, new ExportDataActor.Export()); + return ""; + } + + public Object exportAtags(Request request, Response response) throws Exception { + actorControlService.startFrom(ExecutorActor.EXPORT_ATAGS, new ExportAtagsActor.Export(FileStorageId.parse(request.queryParams("fid")))); + return ""; + } + +}