(control/exec) File listings in control GUI

code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java

@@ -6,6 +6,7 @@ import nu.marginalia.client.AbstractDynamicClient;
 import nu.marginalia.client.Context;
 import nu.marginalia.client.route.RouteProvider;
 import nu.marginalia.client.route.ServiceRoute;
+import nu.marginalia.executor.storage.FileStorageContent;
 import nu.marginalia.service.descriptor.ServiceDescriptor;
 import nu.marginalia.storage.model.FileStorageId;
 import nu.marginalia.executor.model.ActorRunStates;
@@ -69,7 +70,7 @@ public class ExecutorClient extends AbstractDynamicClient {
 
     public void exportData(Context ctx) {
 //        post(ctx, node, "/process/adjacency-calculation/", "").blockingSubscribe();
-        // FIXME
+        // FIXME this shouldn't be done in the executor
     }
 
     public void sideloadEncyclopedia(Context ctx, int node, Path sourcePath) {
@@ -108,4 +109,9 @@ public class ExecutorClient extends AbstractDynamicClient {
     public ActorRunStates getActorStates(Context context, int node) {
         return get(context, node, "/actor", ActorRunStates.class).blockingFirst();
     }
+
+    public FileStorageContent listFileStorage(Context context, int node, FileStorageId fileId) {
+        return get(context, node, "/storage/"+fileId.id(), FileStorageContent.class).blockingFirst();
+    }
+
 }

code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageContent.java

@@ -0,0 +1,7 @@
+package nu.marginalia.executor.storage;
+
+import java.util.List;
+
+public record FileStorageContent(List<FileStorageFile> files)
+{
+}

code/api/executor-api/src/main/java/nu/marginalia/executor/storage/FileStorageFile.java

@@ -0,0 +1,5 @@
+package nu.marginalia.executor.storage;
+
+public record FileStorageFile(String name, long size, String modTime) {
+
+}

code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java

@@ -303,8 +303,8 @@ public class ControlNodeService {
 
     private Object nodeStorageDetailsModel(Request request, Response response) throws SQLException {
         int nodeId = Integer.parseInt(request.params("id"));
+        var storage = getFileStorageWithRelatedEntries(Context.fromRequest(request), nodeId, FileStorageId.parse(request.queryParams("fid")));
 
-        var storage = getFileStorageWithRelatedEntries(FileStorageId.parse(request.queryParams("fid")));
         String view = switch(storage.type()) {
             case BACKUP -> "backup";
             case CRAWL_DATA -> "crawl";
@@ -460,48 +460,37 @@ public class ControlNodeService {
     }
 
 
-    public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(FileStorageId id) throws SQLException {
+    public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(
-        var storage = fileStorageService.getStorage(id);
+            Context context,
-        var related = getRelatedEntries(id);
+            int node,
+            FileStorageId fileId
+    ) throws SQLException {
+        var storage = fileStorageService.getStorage(fileId);
+        var related = getRelatedEntries(fileId);
 
         List<FileStorageFileModel> files = new ArrayList<>();
 
-        try (var filesStream = Files.list(storage.asPath())) {
+        for (var execFile : executorClient.listFileStorage(context, node, fileId).files()) {
-            filesStream
+            files.add(new FileStorageFileModel(
-                    .filter(Files::isRegularFile)
+                    execFile.name(),
-                    .map(this::createFileModel)
+                    execFile.modTime(),
-                    .sorted(Comparator.comparing(FileStorageFileModel::filename))
+                    sizeString(execFile.size())
-                    .forEach(files::add);
+            ));
-        }
-        catch (IOException ex) {
-            logger.error("Failed to list files in storage", ex);
         }
 
         return new FileStorageWithRelatedEntries(new FileStorageWithActions(storage), related, files);
     }
 
-    private FileStorageFileModel createFileModel(Path p) {
+    private String sizeString(long sizeBytes) {
-        try {
+        String size;
-            String mTime = Files.getLastModifiedTime(p).toInstant().toString();
-            String size;
-            if (Files.isDirectory(p)) {
-                size = "-";
-            }
-            else {
-                long sizeBytes = Files.size(p);
 
-                if (sizeBytes < 1024) size = sizeBytes + " B";
+        if (sizeBytes < 1024) size = sizeBytes + " B";
-                else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB";
+        else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB";
-                else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB";
+        else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB";
-                else size = sizeBytes / (1024 * 1024 * 1024) + " GB";
+        else size = sizeBytes / (1024 * 1024 * 1024) + " GB";
-            }
+        return size;
-
-            return new FileStorageFileModel(p.toFile().getName(), mTime, size);
-        }
-        catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
     }
+
     private List<FileStorage> getRelatedEntries(FileStorageId id) {
         List<FileStorage> ret = new ArrayList<>();
         try (var conn = dataSource.getConnection();

code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java

@@ -2,22 +2,34 @@ package nu.marginalia.executor;
 
 import com.google.gson.Gson;
 import com.google.inject.Inject;
+import lombok.SneakyThrows;
 import nu.marginalia.actor.ActorApi;
 import nu.marginalia.actor.ActorControlService;
 import nu.marginalia.actor.state.ActorState;
 import nu.marginalia.actor.state.ActorStateInstance;
 import nu.marginalia.executor.model.ActorRunState;
 import nu.marginalia.executor.model.ActorRunStates;
+import nu.marginalia.executor.storage.FileStorageContent;
+import nu.marginalia.executor.storage.FileStorageFile;
 import nu.marginalia.executor.svc.BackupService;
 import nu.marginalia.executor.svc.ProcessingService;
 import nu.marginalia.executor.svc.SideloadService;
 import nu.marginalia.service.server.BaseServiceParams;
 import nu.marginalia.service.server.Service;
+import nu.marginalia.storage.FileStorageService;
+import nu.marginalia.storage.model.FileStorageId;
 import spark.Request;
 import spark.Response;
 import spark.Spark;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.SQLException;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -25,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap;
 public class ExecutorSvc extends Service {
     private final BaseServiceParams params;
     private final ActorControlService actorControlService;
+    private final FileStorageService fileStorageService;
 
     @Inject
     public ExecutorSvc(BaseServiceParams params,
@@ -32,11 +45,13 @@ public class ExecutorSvc extends Service {
                        ProcessingService processingService,
                        SideloadService sideloadService,
                        BackupService backupService,
+                       FileStorageService fileStorageService,
                        Gson gson,
                        ActorApi actorApi) {
         super(params);
         this.params = params;
         this.actorControlService = actorControlService;
+        this.fileStorageService = fileStorageService;
 
         Spark.post("/actor/:id/start", actorApi::startActor);
         Spark.post("/actor/:id/start/:state", actorApi::startActorFromState);
@@ -57,9 +72,36 @@ public class ExecutorSvc extends Service {
         Spark.post("/sideload/encyclopedia", sideloadService::sideloadEncyclopedia);
 
         Spark.post("/backup/:fid/restore", backupService::restore);
+        Spark.get("/storage/:fid", this::listFiles, gson::toJson);
 
     }
 
+    private FileStorageContent listFiles(Request request, Response response) throws SQLException, IOException {
+        FileStorageId fileStorageId = FileStorageId.parse(request.params("fid"));
+
+        var storage = fileStorageService.getStorage(fileStorageId);
+
+        List<FileStorageFile> files;
+
+        try (var fs = Files.list(storage.asPath())) {
+            files = fs.filter(Files::isRegularFile)
+                    .map(this::createFileModel)
+                    .sorted(Comparator.comparing(FileStorageFile::name))
+                    .toList();
+        }
+
+        return new FileStorageContent(files);
+    }
+
+    @SneakyThrows
+    private FileStorageFile createFileModel(Path path) {
+        return new FileStorageFile(
+                path.toFile().getName(),
+                Files.size(path),
+                Files.getLastModifiedTime(path).toInstant().toString()
+                );
+    }
+
 
     private final ConcurrentHashMap<String, String> actorStateDescriptions = new ConcurrentHashMap<>();