diff --git a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java
index b5554edd..df9c4072 100644
--- a/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java
+++ b/code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java
@@ -71,7 +71,6 @@ public class ExecutorClient extends AbstractDynamicClient {
post(ctx, node,
"/sideload/encyclopedia?path="+ URLEncoder.encode(sourcePath.toString(), StandardCharsets.UTF_8) + "&baseUrl=" + URLEncoder.encode(baseUrl, StandardCharsets.UTF_8),
"").blockingSubscribe();
-
}
public void sideloadDirtree(Context ctx, int node, Path sourcePath) {
@@ -111,6 +110,10 @@ public class ExecutorClient extends AbstractDynamicClient {
post(ctx, node, "/export/termfreq?fid="+fid, "").blockingSubscribe();
}
+ public void downloadSampleData(Context ctx, int node, String sampleSet) {
+ post(ctx, node, "/action/download-sample-data?set="+URLEncoder.encode(sampleSet, StandardCharsets.UTF_8), "").blockingSubscribe();
+ }
+
public void exportData(Context ctx, int node) {
post(ctx, node, "/export/data", "").blockingSubscribe();
}
@@ -166,4 +169,5 @@ public class ExecutorClient extends AbstractDynamicClient {
public void yieldDomain(Context context, int node, TransferItem item) {
post(context, node, "/transfer/yield", item).blockingSubscribe();
}
+
}
diff --git a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java
index 999a51a5..b2162664 100644
--- a/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java
+++ b/code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java
@@ -223,14 +223,12 @@ public class FileStorageService {
return maybePath;
}
- /** Allocate a temporary storage of the given type */
- public FileStorage allocateTemporaryStorage(FileStorageBase base,
- FileStorageType type,
- String prefix,
- String description) throws IOException, SQLException
+ /** Allocate a storage area of the given type */
+ public FileStorage allocateStorage(FileStorageType type,
+ String prefix,
+ String description) throws IOException, SQLException
{
- if (!base.type().permitsStorageType(type))
- throw new RuntimeException("Attempting to allocate storage of type " + type + " in base of type " + base.type());
+ var base = getStorageBase(FileStorageBaseType.forFileStorageType(type));
Path newDir = allocateDirectory(base.asPath(), prefix);
diff --git a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java
index 7bd1eb0d..d319786f 100644
--- a/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java
+++ b/code/common/config/src/main/java/nu/marginalia/storage/model/FileStorageBaseType.java
@@ -1,18 +1,17 @@
package nu.marginalia.storage.model;
-import java.util.EnumSet;
-
public enum FileStorageBaseType {
CURRENT,
WORK,
STORAGE,
BACKUP;
- public boolean permitsStorageType(FileStorageType type) {
- return switch (this) {
- case BACKUP -> FileStorageType.BACKUP.equals(type);
- case STORAGE -> EnumSet.of(FileStorageType.EXPORT, FileStorageType.CRAWL_DATA, FileStorageType.PROCESSED_DATA, FileStorageType.CRAWL_SPEC).contains(type);
- default -> false;
+
+ public static FileStorageBaseType forFileStorageType(FileStorageType type) {
+ return switch (type) {
+ case EXPORT, CRAWL_DATA, PROCESSED_DATA, CRAWL_SPEC -> STORAGE;
+ case BACKUP -> BACKUP;
};
}
+
}
diff --git a/code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java b/code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java
index 69fa9b1b..c9ffa309 100644
--- a/code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java
+++ b/code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java
@@ -13,18 +13,14 @@ import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Objects;
import java.util.UUID;
-import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
-
@Testcontainers
@Execution(ExecutionMode.SAME_THREAD)
@Tag("slow")
@@ -124,8 +120,7 @@ public class FileStorageServiceTest {
var storage = new FileStorageService(dataSource, 0);
- var base = storage.createStorageBase(name, createTempDir(), FileStorageBaseType.STORAGE);
- var fileStorage = storage.allocateTemporaryStorage(base, FileStorageType.CRAWL_DATA, "xyz", "thisShouldSucceed");
+ var fileStorage = storage.allocateStorage(FileStorageType.CRAWL_DATA, "xyz", "thisShouldSucceed");
System.out.println("Allocated " + fileStorage.asPath());
Assertions.assertTrue(Files.exists(fileStorage.asPath()));
tempDirs.add(fileStorage.asPath());
diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java
index 16b06998..a707aefe 100644
--- a/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java
+++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java
@@ -13,6 +13,8 @@ import nu.marginalia.storage.FileStorageService;
import nu.marginalia.storage.model.FileStorageId;
import nu.marginalia.storage.model.FileStorageState;
import nu.marginalia.storage.model.FileStorageType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import spark.Request;
import spark.Response;
import spark.Spark;
@@ -21,9 +23,11 @@ import java.nio.file.Path;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
+import java.util.Set;
@Singleton
public class ControlNodeActionsService {
+ private static final Logger logger = LoggerFactory.getLogger(ControlNodeActionsService.class);
private final IndexClient indexClient;
private final RedirectControl redirectControl;
private final FileStorageService fileStorageService;
@@ -62,6 +66,9 @@ public class ControlNodeActionsService {
Spark.post("/public/nodes/:node/actions/sideload-stackexchange", this::sideloadStackexchange,
redirectControl.renderRedirectAcknowledgement("Sideloading", "..")
);
+ Spark.post("/public/nodes/:node/actions/download-sample-data", this::downloadSampleData,
+ redirectControl.renderRedirectAcknowledgement("Downloading", "..")
+ );
Spark.post("/public/nodes/:id/actions/new-crawl", this::triggerNewCrawl,
redirectControl.renderRedirectAcknowledgement("Crawling", "..")
);
@@ -91,6 +98,21 @@ public class ControlNodeActionsService {
);
}
+ private Object downloadSampleData(Request request, Response response) {
+ String set = request.queryParams("sample");
+
+ if (set == null)
+ throw new ControlValidationError("No sample specified", "A sample data set must be specified", "..");
+ if (!Set.of("sample-s", "sample-m", "sample-l", "sample-xl").contains(set))
+ throw new ControlValidationError("Invalid sample specified", "A valid sample data set must be specified", "..");
+
+ executorClient.downloadSampleData(Context.fromRequest(request), Integer.parseInt(request.params("node")), set);
+
+ logger.info("Downloading sample data set {}", set);
+
+ return "";
+ }
+
public Object sideloadEncyclopedia(Request request, Response response) {
String source = request.queryParams("source");
diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-download-sample-data.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-download-sample-data.hdb
new file mode 100644
index 00000000..3c432e65
--- /dev/null
+++ b/code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-download-sample-data.hdb
@@ -0,0 +1,47 @@
+
Download Sample Data
+
+
+This will download sample crawl data from downloads.marginalia.nu onto Node {{node.id}}.
+This is a sample of real crawl data. It is intended for demo, testing and development purposes. Several sets are available.
+
+
+
\ No newline at end of file
diff --git a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb
index 736c7961..ea4502fa 100644
--- a/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb
+++ b/code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb
@@ -23,6 +23,7 @@
{{#if view.export-from-crawl-data}} {{> control/node/actions/partial-export-from-crawl-data }} {{/if}}
{{#if view.export-sample-data}} {{> control/node/actions/partial-export-sample-data }} {{/if}}
{{#if view.restore-backup}} {{> control/node/actions/partial-restore-backup }} {{/if}}
+ {{#if view.download-sample-data}} {{> control/node/actions/partial-download-sample-data }} {{/if}}