diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java index 0fb26715..d174569a 100644 --- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java @@ -6,6 +6,7 @@ import nu.marginalia.client.AbstractDynamicClient; import nu.marginalia.client.Context; import nu.marginalia.client.route.RouteProvider; import nu.marginalia.client.route.ServiceRoute; +import nu.marginalia.executor.storage.FileStorageContent; import nu.marginalia.service.descriptor.ServiceDescriptor; import nu.marginalia.storage.model.FileStorageId; import nu.marginalia.executor.model.ActorRunStates; @@ -69,7 +70,7 @@ public class ExecutorClient extends AbstractDynamicClient { public void exportData(Context ctx) { // post(ctx, node, "/process/adjacency-calculation/", "").blockingSubscribe(); - // FIXME + // FIXME this shouldn't be done in the executor } public void sideloadEncyclopedia(Context ctx, int node, Path sourcePath) { @@ -108,4 +109,9 @@ public class ExecutorClient extends AbstractDynamicClient { public ActorRunStates getActorStates(Context context, int node) { return get(context, node, "/actor", ActorRunStates.class).blockingFirst(); } + + public FileStorageContent listFileStorage(Context context, int node, FileStorageId fileId) { + return get(context, node, "/storage/"+fileId.id(), FileStorageContent.class).blockingFirst(); + } + } diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java new file mode 100644 index 00000000..7b747217 --- /dev/null +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java @@ -0,0 +1,7 @@ +package nu.marginalia.executor.storage; + +import java.util.List; + +public record FileStorageContent(List files) +{ +} diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java new file mode 100644 index 00000000..ff40e60d --- /dev/null +++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java @@ -0,0 +1,5 @@ +package nu.marginalia.executor.storage; + +public record FileStorageFile(String name, long size, String modTime) { + +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java index 74889b65..f1af8de1 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java @@ -303,8 +303,8 @@ public class ControlNodeService { private Object nodeStorageDetailsModel(Request request, Response response) throws SQLException { int nodeId = Integer.parseInt(request.params("id")); + var storage = getFileStorageWithRelatedEntries(Context.fromRequest(request), nodeId, FileStorageId.parse(request.queryParams("fid"))); - var storage = getFileStorageWithRelatedEntries(FileStorageId.parse(request.queryParams("fid"))); String view = switch(storage.type()) { case BACKUP -> "backup"; case CRAWL_DATA -> "crawl"; @@ -460,48 +460,37 @@ public class ControlNodeService { } - public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(FileStorageId id) throws SQLException { - var storage = fileStorageService.getStorage(id); - var related = getRelatedEntries(id); + public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries( + Context context, + int node, + FileStorageId fileId + ) throws SQLException { + var storage = fileStorageService.getStorage(fileId); + var related = getRelatedEntries(fileId); List files = new ArrayList<>(); - try (var filesStream = Files.list(storage.asPath())) { - filesStream - .filter(Files::isRegularFile) - .map(this::createFileModel) - .sorted(Comparator.comparing(FileStorageFileModel::filename)) - .forEach(files::add); - } - catch (IOException ex) { - logger.error("Failed to list files in storage", ex); + for (var execFile : executorClient.listFileStorage(context, node, fileId).files()) { + files.add(new FileStorageFileModel( + execFile.name(), + execFile.modTime(), + sizeString(execFile.size()) + )); } return new FileStorageWithRelatedEntries(new FileStorageWithActions(storage), related, files); } - private FileStorageFileModel createFileModel(Path p) { - try { - String mTime = Files.getLastModifiedTime(p).toInstant().toString(); - String size; - if (Files.isDirectory(p)) { - size = "-"; - } - else { - long sizeBytes = Files.size(p); + private String sizeString(long sizeBytes) { + String size; - if (sizeBytes < 1024) size = sizeBytes + " B"; - else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB"; - else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB"; - else size = sizeBytes / (1024 * 1024 * 1024) + " GB"; - } - - return new FileStorageFileModel(p.toFile().getName(), mTime, size); - } - catch (IOException ex) { - throw new RuntimeException(ex); - } + if (sizeBytes < 1024) size = sizeBytes + " B"; + else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB"; + else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB"; + else size = sizeBytes / (1024 * 1024 * 1024) + " GB"; + return size; } + private List getRelatedEntries(FileStorageId id) { List ret = new ArrayList<>(); try (var conn = dataSource.getConnection(); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java index a7a7b1db..130ce23d 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java @@ -2,22 +2,34 @@ package nu.marginalia.executor; import com.google.gson.Gson; import com.google.inject.Inject; +import lombok.SneakyThrows; import nu.marginalia.actor.ActorApi; import nu.marginalia.actor.ActorControlService; import nu.marginalia.actor.state.ActorState; import nu.marginalia.actor.state.ActorStateInstance; import nu.marginalia.executor.model.ActorRunState; import nu.marginalia.executor.model.ActorRunStates; +import nu.marginalia.executor.storage.FileStorageContent; +import nu.marginalia.executor.storage.FileStorageFile; import nu.marginalia.executor.svc.BackupService; import nu.marginalia.executor.svc.ProcessingService; import nu.marginalia.executor.svc.SideloadService; import nu.marginalia.service.server.BaseServiceParams; import nu.marginalia.service.server.Service; +import nu.marginalia.storage.FileStorageService; +import nu.marginalia.storage.model.FileStorageId; import spark.Request; import spark.Response; import spark.Spark; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -25,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; public class ExecutorSvc extends Service { private final BaseServiceParams params; private final ActorControlService actorControlService; + private final FileStorageService fileStorageService; @Inject public ExecutorSvc(BaseServiceParams params, @@ -32,11 +45,13 @@ public class ExecutorSvc extends Service { ProcessingService processingService, SideloadService sideloadService, BackupService backupService, + FileStorageService fileStorageService, Gson gson, ActorApi actorApi) { super(params); this.params = params; this.actorControlService = actorControlService; + this.fileStorageService = fileStorageService; Spark.post("/actor/:id/start", actorApi::startActor); Spark.post("/actor/:id/start/:state", actorApi::startActorFromState); @@ -57,9 +72,36 @@ public class ExecutorSvc extends Service { Spark.post("/sideload/encyclopedia", sideloadService::sideloadEncyclopedia); Spark.post("/backup/:fid/restore", backupService::restore); + Spark.get("/storage/:fid", this::listFiles, gson::toJson); } + private FileStorageContent listFiles(Request request, Response response) throws SQLException, IOException { + FileStorageId fileStorageId = FileStorageId.parse(request.params("fid")); + + var storage = fileStorageService.getStorage(fileStorageId); + + List files; + + try (var fs = Files.list(storage.asPath())) { + files = fs.filter(Files::isRegularFile) + .map(this::createFileModel) + .sorted(Comparator.comparing(FileStorageFile::name)) + .toList(); + } + + return new FileStorageContent(files); + } + + @SneakyThrows + private FileStorageFile createFileModel(Path path) { + return new FileStorageFile( + path.toFile().getName(), + Files.size(path), + Files.getLastModifiedTime(path).toInstant().toString() + ); + } + private final ConcurrentHashMap actorStateDescriptions = new ConcurrentHashMap<>();