From 66bb12e55a4be2adbfbdd74d0bbe331bd5c8cb41 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Wed, 26 Jul 2023 21:59:35 +0200 Subject: [PATCH] (converter) File listing and download for file storage --- .../nu/marginalia/control/ControlService.java | 1 + .../control/model/FileStorageFileModel.java | 15 ++++ .../model/FileStorageWithRelatedEntries.java | 5 +- .../svc/ControlFileStorageService.java | 78 +++++++++++++++++-- .../templates/control/storage-details.hdb | 23 +++++- 5 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageFileModel.java diff --git a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/ControlService.java index cc2e74fd..3411058c 100644 --- a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -85,6 +85,7 @@ public class ControlService extends Service { Spark.get("/public/storage/crawls", this::storageModelCrawls, storageCrawlsRenderer::render); Spark.get("/public/storage/processed", this::storageModelProcessed, storageProcessedRenderer::render); Spark.get("/public/storage/:id", this::storageDetailsModel, storageDetailsRenderer::render); + Spark.get("/public/storage/:id/file", controlFileStorageService::downloadFileFromStorage); final HtmlRedirect redirectToServices = new HtmlRedirect("/services"); diff --git a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageFileModel.java b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageFileModel.java new file mode 100644 index 00000000..c8b513ee --- /dev/null +++ b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageFileModel.java @@ -0,0 +1,15 @@ +package nu.marginalia.control.model; + +import nu.marginalia.db.storage.model.FileStorage; + +import java.util.List; + +public record FileStorageFileModel(String filename, + String type, + String size + ) { + + public boolean isDownloadable() { + return type.equals("file"); + } +} diff --git a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageWithRelatedEntries.java b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageWithRelatedEntries.java index 28afba5d..608ccdca 100644 --- a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageWithRelatedEntries.java +++ b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/model/FileStorageWithRelatedEntries.java @@ -5,6 +5,9 @@ import nu.marginalia.db.storage.model.FileStorageType; import java.util.List; -public record FileStorageWithRelatedEntries(FileStorageWithActions self, List related) { +public record FileStorageWithRelatedEntries(FileStorageWithActions self, + List related, + List files + ) { } diff --git a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java index db122a7c..06bf240d 100644 --- a/code/services-satellite/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java +++ b/code/services-satellite/control-service/src/main/java/nu/marginalia/control/svc/ControlFileStorageService.java @@ -7,19 +7,23 @@ import lombok.SneakyThrows; import nu.marginalia.control.model.*; import nu.marginalia.db.storage.FileStorageService; import nu.marginalia.db.storage.model.*; +import nu.marginalia.mqsm.graph.AbstractStateGraph; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Singleton public class ControlFileStorageService { private final HikariDataSource dataSource; private final FileStorageService fileStorageService; + private Logger logger = LoggerFactory.getLogger(getClass()); @Inject public ControlFileStorageService(HikariDataSource dataSource, FileStorageService fileStorageService) { @@ -107,9 +111,47 @@ public class ControlFileStorageService { public FileStorageWithRelatedEntries getFileStorageWithRelatedEntries(FileStorageId id) throws SQLException { var storage = fileStorageService.getStorage(id); var related = getRelatedEntries(id); - return new FileStorageWithRelatedEntries(new FileStorageWithActions(storage), related); + + List files = new ArrayList<>(); + + try (var filesStream = Files.list(storage.asPath())) { + filesStream + .map(this::createFileModel) + .sorted(Comparator + .comparing(FileStorageFileModel::type) + .thenComparing(FileStorageFileModel::filename) + ) + .forEach(files::add); + } + catch (IOException ex) { + logger.error("Failed to list files in storage", ex); + } + + return new FileStorageWithRelatedEntries(new FileStorageWithActions(storage), related, files); } + private FileStorageFileModel createFileModel(Path p) { + try { + String type = Files.isRegularFile(p) ? "file" : "directory"; + String size; + if (Files.isDirectory(p)) { + size = "-"; + } + else { + long sizeBytes = Files.size(p); + + if (sizeBytes < 1024) size = sizeBytes + " B"; + else if (sizeBytes < 1024 * 1024) size = sizeBytes / 1024 + " KB"; + else if (sizeBytes < 1024 * 1024 * 1024) size = sizeBytes / (1024 * 1024) + " MB"; + else size = sizeBytes / (1024 * 1024 * 1024) + " GB"; + } + + return new FileStorageFileModel(p.toFile().getName(), type, size); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } private List getRelatedEntries(FileStorageId id) { List ret = new ArrayList<>(); try (var conn = dataSource.getConnection(); @@ -131,4 +173,30 @@ public class ControlFileStorageService { } return ret; } + + public Object downloadFileFromStorage(Request request, Response response) throws SQLException { + var fileStorageId = FileStorageId.parse(request.params("id")); + String filename = request.queryParams("name"); + + Path root = fileStorageService.getStorage(fileStorageId).asPath(); + Path filePath = root.resolve(filename).normalize(); + + if (!filePath.startsWith(root)) { + response.status(403); + return ""; + } + + if (filePath.endsWith(".txt") || filePath.endsWith(".log")) response.type("text/plain"); + else response.type("application/octet-stream"); + + try (var is = Files.newInputStream(filePath)) { + is.transferTo(response.raw().getOutputStream()); + } + catch (IOException ex) { + logger.error("Failed to download file", ex); + throw new RuntimeException(ex); + } + + return ""; + } } diff --git a/code/services-satellite/control-service/src/main/resources/templates/control/storage-details.hdb b/code/services-satellite/control-service/src/main/resources/templates/control/storage-details.hdb index 9038d510..ec7d4ef0 100644 --- a/code/services-satellite/control-service/src/main/resources/templates/control/storage-details.hdb +++ b/code/services-satellite/control-service/src/main/resources/templates/control/storage-details.hdb @@ -24,7 +24,28 @@ {{/with}} -

Actions

+ + {{#if storage.files}} +

Contents

+ + + + + + + {{#each storage.files}} + + + {{else}} {{filename}} {{/if}} + + + + {{/each}} +
File NameTypeSize
+ {{#if downloadable}}{{filename}}{{type}}{{size}}
+ {{/if}} + +

Actions

{{#with storage.self}} {{#if isCrawlable}}