(control) UX-improvements for control service

This commit overhauls a lot of the UX for the control service, adding a new actions menu to the nodes views.  It has many small tweaks to make the work flow better.

It also adds a new /uploads directory in each index node, from which sideloaded data can be selected.  This is a bit of a breaking change, as this directory needs to exist in each index node.

code/api/executor-api/src/main/java/nu/marginalia/executor/client/ExecutorClient.java

@@ -8,6 +8,7 @@ import nu.marginalia.executor.model.load.LoadParameters;
 import nu.marginalia.executor.model.transfer.TransferItem;
 import nu.marginalia.executor.model.transfer.TransferSpec;
 import nu.marginalia.executor.storage.FileStorageContent;
+import nu.marginalia.executor.upload.UploadDirContents;
 import nu.marginalia.model.gson.GsonFactory;
 import nu.marginalia.service.descriptor.ServiceDescriptors;
 import nu.marginalia.service.id.ServiceId;
@@ -38,7 +39,7 @@ public class ExecutorClient extends AbstractDynamicClient {
     }
 
 
-    public void triggerCrawl(Context ctx, int node, String fid) {
+    public void triggerCrawl(Context ctx, int node, FileStorageId fid) {
         post(ctx, node, "/process/crawl/" + fid, "").blockingSubscribe();
     }
 
@@ -112,6 +113,10 @@ public class ExecutorClient extends AbstractDynamicClient {
         return get(context, node, "/actor", ActorRunStates.class).blockingFirst();
     }
 
+    public UploadDirContents listSideloadDir(Context context, int node) {
+        return get(context, node, "/sideload/", UploadDirContents.class).blockingFirst();
+    }
+
     public FileStorageContent listFileStorage(Context context, int node, FileStorageId fileId) {
         return get(context, node, "/storage/"+fileId.id(), FileStorageContent.class).blockingFirst();
     }

code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirContents.java

@@ -0,0 +1,6 @@
+package nu.marginalia.executor.upload;
+
+import java.util.List;
+
+public record UploadDirContents(String path, List<UploadDirItem> items) {
+}

code/api/executor-api/src/main/java/nu/marginalia/executor/upload/UploadDirItem.java

@@ -0,0 +1,29 @@
+package nu.marginalia.executor.upload;
+
+import lombok.SneakyThrows;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+public record UploadDirItem (
+        String name,
+        String lastModifiedTime,
+        boolean isDirectory,
+        long size
+) {
+
+    @SneakyThrows
+    public static UploadDirItem fromPath(Path path) {
+        boolean isDir = Files.isDirectory(path);
+        long size = isDir ? 0 : Files.size(path);
+        var mtime = Files.getLastModifiedTime(path);
+
+
+        return new UploadDirItem(path.toString(),
+                LocalDateTime.ofInstant(mtime.toInstant(), ZoneId.systemDefault()).format(DateTimeFormatter.ISO_DATE_TIME), isDir, size);
+    }
+
+}

code/common/config/src/main/java/nu/marginalia/WmsaHome.java

@@ -23,6 +23,10 @@ public class WmsaHome {
     }
 
 
+    public static Path getUploadDir() {
+        return Path.of("/uploads");
+    }
+
     public static Path getHomePath() {
         var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElseGet(WmsaHome::findDefaultHomePath);
 

code/common/config/src/main/java/nu/marginalia/storage/FileStorageService.java

@@ -455,6 +455,57 @@ public class FileStorageService {
         return ret;
     }
 
+    public List<FileStorage> getEachFileStorage(FileStorageType type) {
+        return getEachFileStorage(node, type);
+    }
+
+    public List<FileStorage> getEachFileStorage(int node, FileStorageType type) {
+        List<FileStorage> ret = new ArrayList<>();
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT PATH, STATE, TYPE, DESCRIPTION, CREATE_DATE, ID, BASE_ID
+                     FROM FILE_STORAGE_VIEW
+                     WHERE NODE=? AND TYPE=?
+                     """)) {
+
+            stmt.setInt(1, node);
+            stmt.setString(2, type.name());
+
+            long storageId;
+            long baseId;
+            String path;
+            String state;
+            String description;
+            LocalDateTime createDateTime;
+
+            try (var rs = stmt.executeQuery()) {
+                while (rs.next()) {
+                    baseId = rs.getLong("BASE_ID");
+                    storageId = rs.getLong("ID");
+                    path = rs.getString("PATH");
+                    state = rs.getString("STATE");
+
+                    description = rs.getString("DESCRIPTION");
+                    createDateTime = rs.getTimestamp("CREATE_DATE").toLocalDateTime();
+                    var base = getStorageBase(new FileStorageBaseId(baseId));
+
+                    ret.add(new FileStorage(
+                            new FileStorageId(storageId),
+                            base,
+                            type,
+                            createDateTime,
+                            path,
+                            FileStorageState.parse(state),
+                            description
+                    ));
+                }
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+
+        return ret;
+    }
     public void flagFileForDeletion(FileStorageId id) throws SQLException {
         setFileStorageState(id, FileStorageState.DELETE);
     }

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlHandlebarsConfigurator.java

@@ -1,10 +1,10 @@
 package nu.marginalia.control;
 
-import com.github.jknack.handlebars.Handlebars;
+import com.github.jknack.handlebars.*;
-import com.github.jknack.handlebars.Helper;
-import com.github.jknack.handlebars.Options;
 import nu.marginalia.renderer.config.HandlebarsConfigurator;
 
+import java.io.IOException;
+
 public class ControlHandlebarsConfigurator implements HandlebarsConfigurator {
     @Override
     public void configure(Handlebars handlebars) {

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlRendererFactory.java

@@ -0,0 +1,38 @@
+package nu.marginalia.control;
+
+import com.google.inject.Inject;
+import lombok.SneakyThrows;
+import nu.marginalia.nodecfg.NodeConfigurationService;
+import nu.marginalia.renderer.RendererFactory;
+
+import java.util.Map;
+
+/** Wrapper for the renderer factory that adds global context
+ * with the nodes listing
+ */
+public class ControlRendererFactory {
+    private final RendererFactory rendererFactory;
+    private final NodeConfigurationService nodeConfigurationService;
+
+    @Inject
+    public ControlRendererFactory(RendererFactory rendererFactory,
+                                  NodeConfigurationService nodeConfigurationService)
+    {
+        this.rendererFactory = rendererFactory;
+        this.nodeConfigurationService = nodeConfigurationService;
+    }
+
+    @SneakyThrows
+    public Renderer renderer(String template) {
+        Map<String, Object> globalContext = Map.of(
+                "nodes", nodeConfigurationService.getAll()
+        );
+        var baseRenderer = rendererFactory.renderer(template);
+
+        return (context) -> baseRenderer.render(context, Map.of("global-context", globalContext));
+    }
+
+    public interface Renderer {
+        String render(Object context);
+    }
+}

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -10,7 +10,6 @@ import nu.marginalia.control.node.svc.ControlFileStorageService;
 import nu.marginalia.control.node.svc.ControlNodeService;
 import nu.marginalia.control.sys.svc.*;
 import nu.marginalia.model.gson.GsonFactory;
-import nu.marginalia.renderer.RendererFactory;
 import nu.marginalia.screenshot.ScreenshotService;
 import nu.marginalia.service.server.*;
 import org.slf4j.Logger;
@@ -40,7 +39,7 @@ public class ControlService extends Service {
                           ServiceMonitors monitors,
                           HeartbeatService heartbeatService,
                           EventLogService eventLogService,
-                          RendererFactory rendererFactory,
+                          ControlRendererFactory rendererFactory,
                           StaticResources staticResources,
                           MessageQueueService messageQueueService,
                           ControlFileStorageService controlFileStorageService,

code/services-core/control-service/src/main/java/nu/marginalia/control/RedirectControl.java

@@ -1,18 +1,16 @@
 package nu.marginalia.control;
 
 import jakarta.inject.Inject;
-import nu.marginalia.renderer.MustacheRenderer;
-import nu.marginalia.renderer.RendererFactory;
 import spark.ResponseTransformer;
 
 import java.io.IOException;
 import java.util.Map;
 
 public class RedirectControl {
-    private final MustacheRenderer<Object> renderer;
+    private final ControlRendererFactory.Renderer renderer;
 
     @Inject
-    public RedirectControl(RendererFactory rendererFactory) throws IOException {
+    public RedirectControl(ControlRendererFactory rendererFactory) throws IOException {
         renderer = rendererFactory.renderer("control/redirect-ok");
     }
 

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ApiKeyService.java

@@ -2,9 +2,9 @@ package nu.marginalia.control.app.svc;
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.app.model.ApiKeyModel;
-import nu.marginalia.renderer.RendererFactory;
 import org.eclipse.jetty.util.StringUtil;
 import spark.Request;
 import spark.Response;
@@ -20,11 +20,11 @@ import java.util.UUID;
 public class ApiKeyService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
 
     @Inject
     public ApiKeyService(HikariDataSource dataSource,
-                         RendererFactory rendererFactory
+                         ControlRendererFactory rendererFactory
                          ) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/ControlBlacklistService.java

@@ -2,10 +2,10 @@ package nu.marginalia.control.app.svc;
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.app.model.BlacklistedDomainModel;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.renderer.RendererFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -22,12 +22,12 @@ import java.util.Objects;
 public class ControlBlacklistService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Inject
     public ControlBlacklistService(HikariDataSource dataSource,
-                                   RendererFactory rendererFactory) {
+                                   ControlRendererFactory rendererFactory) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;
     }

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/DomainComplaintService.java

@@ -3,11 +3,11 @@ package nu.marginalia.control.app.svc;
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.app.model.DomainComplaintCategory;
 import nu.marginalia.control.app.model.DomainComplaintModel;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.renderer.RendererFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -26,14 +26,14 @@ import java.util.stream.Collectors;
  */
 public class DomainComplaintService {
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final ControlBlacklistService blacklistService;
     private final RandomExplorationService randomExplorationService;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Inject
     public DomainComplaintService(HikariDataSource dataSource,
-                                  RendererFactory rendererFactory,
+                                  ControlRendererFactory rendererFactory,
                                   ControlBlacklistService blacklistService,
                                   RandomExplorationService randomExplorationService
     ) {

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/RandomExplorationService.java

@@ -3,8 +3,8 @@ package nu.marginalia.control.app.svc;
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import gnu.trove.list.array.TIntArrayList;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.renderer.RendererFactory;
 import spark.Request;
 import spark.Response;
 import spark.Spark;
@@ -19,11 +19,11 @@ import java.util.Objects;
 public class RandomExplorationService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
 
     @Inject
     public RandomExplorationService(HikariDataSource dataSource,
-                                    RendererFactory rendererFactory
+                                    ControlRendererFactory rendererFactory
     ) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;
@@ -33,6 +33,7 @@ public class RandomExplorationService {
         var reviewRandomDomainsRenderer = rendererFactory.renderer("control/app/review-random-domains");
 
         Spark.get("/public/review-random-domains", this::reviewRandomDomainsModel, reviewRandomDomainsRenderer::render);
+
         Spark.post("/public/review-random-domains", this::reviewRandomDomainsAction);
     }
 

code/services-core/control-service/src/main/java/nu/marginalia/control/app/svc/SearchToBanService.java

@@ -2,12 +2,13 @@ package nu.marginalia.control.app.svc;
 
 import com.google.inject.Inject;
 import nu.marginalia.client.Context;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.index.client.model.query.SearchSetIdentifier;
 import nu.marginalia.index.query.limit.QueryLimits;
 import nu.marginalia.model.EdgeUrl;
+import nu.marginalia.nodecfg.NodeConfigurationService;
 import nu.marginalia.query.client.QueryClient;
 import nu.marginalia.query.model.QueryParams;
-import nu.marginalia.renderer.RendererFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -20,19 +21,21 @@ import java.util.Objects;
 
 public class SearchToBanService {
     private final ControlBlacklistService blacklistService;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final QueryClient queryClient;
     private final Logger logger = LoggerFactory.getLogger(getClass());
+    private final NodeConfigurationService nodeConfigurationService;
 
     @Inject
     public SearchToBanService(ControlBlacklistService blacklistService,
-                              RendererFactory rendererFactory,
+                              ControlRendererFactory rendererFactory,
-                              QueryClient queryClient)
+                              QueryClient queryClient, NodeConfigurationService nodeConfigurationService)
     {
 
         this.blacklistService = blacklistService;
         this.rendererFactory = rendererFactory;
         this.queryClient = queryClient;
+        this.nodeConfigurationService = nodeConfigurationService;
     }
 
     public void register() throws IOException {

code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeActionsService.java

@@ -5,20 +5,27 @@ import com.google.inject.Singleton;
 import nu.marginalia.client.Context;
 import nu.marginalia.control.RedirectControl;
 import nu.marginalia.executor.client.ExecutorClient;
+import nu.marginalia.executor.model.load.LoadParameters;
 import nu.marginalia.index.client.IndexClient;
 import nu.marginalia.service.control.ServiceEventLog;
+import nu.marginalia.storage.FileStorageService;
+import nu.marginalia.storage.model.FileStorageId;
+import nu.marginalia.storage.model.FileStorageState;
+import nu.marginalia.storage.model.FileStorageType;
 import spark.Request;
 import spark.Response;
 import spark.Spark;
 
-import java.nio.file.Files;
 import java.nio.file.Path;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.List;
 
 @Singleton
 public class ControlNodeActionsService {
-
     private final IndexClient indexClient;
     private final RedirectControl redirectControl;
+    private final FileStorageService fileStorageService;
     private final ServiceEventLog eventLog;
     private final ExecutorClient executorClient;
 
@@ -26,12 +33,14 @@ public class ControlNodeActionsService {
     public ControlNodeActionsService(ExecutorClient executorClient,
                                      IndexClient indexClient,
                                      RedirectControl redirectControl,
+                                     FileStorageService fileStorageService,
                                      ServiceEventLog eventLog)
     {
         this.executorClient = executorClient;
 
         this.indexClient = indexClient;
         this.redirectControl = redirectControl;
+        this.fileStorageService = fileStorageService;
         this.eventLog = eventLog;
 
     }
@@ -52,15 +61,24 @@ public class ControlNodeActionsService {
         Spark.post("/public/nodes/:node/actions/sideload-stackexchange", this::sideloadStackexchange,
                 redirectControl.renderRedirectAcknowledgement("Sideloading", "..")
         );
+        Spark.post("/public/nodes/:id/actions/new-crawl", this::triggerNewCrawl,
+                redirectControl.renderRedirectAcknowledgement("Crawling", ".."));
+        Spark.post("/public/nodes/:id/actions/recrawl", this::triggerAutoRecrawl,
+                redirectControl.renderRedirectAcknowledgement("Recrawling", ".."));
+        Spark.post("/public/nodes/:id/actions/process", this::triggerAutoProcess,
+                redirectControl.renderRedirectAcknowledgement("Processing", "..")
+        );
+        Spark.post("/public/nodes/:id/actions/load", this::triggerLoadSelected,
+                redirectControl.renderRedirectAcknowledgement("Loading", "..")
+        );
+        Spark.post("/public/nodes/:id/actions/restore-backup", this::triggerRestoreBackup,
+                redirectControl.renderRedirectAcknowledgement("Restoring", "..")
+        );
     }
 
     public Object sideloadEncyclopedia(Request request, Response response) throws Exception {
 
         Path sourcePath = Path.of(request.queryParams("source"));
-        if (!Files.exists(sourcePath)) {
-            Spark.halt(404);
-            return "No such file " + sourcePath;
-        }
         String baseUrl = request.queryParams("baseUrl");
 
         final int nodeId = Integer.parseInt(request.params("node"));
@@ -75,11 +93,6 @@ public class ControlNodeActionsService {
     public Object sideloadDirtree(Request request, Response response) throws Exception {
 
         Path sourcePath = Path.of(request.queryParams("source"));
-        if (!Files.exists(sourcePath)) {
-            Spark.halt(404);
-            return "No such file " + sourcePath;
-        }
-
         final int nodeId = Integer.parseInt(request.params("node"));
 
         eventLog.logEvent("USER-ACTION", "SIDELOAD DIRTREE " + nodeId);
@@ -92,11 +105,6 @@ public class ControlNodeActionsService {
     public Object sideloadWarc(Request request, Response response) throws Exception {
 
         Path sourcePath = Path.of(request.queryParams("source"));
-        if (!Files.exists(sourcePath)) {
-            Spark.halt(404);
-            return "No such file " + sourcePath;
-        }
-
         final int nodeId = Integer.parseInt(request.params("node"));
 
         eventLog.logEvent("USER-ACTION", "SIDELOAD WARC " + nodeId);
@@ -108,11 +116,6 @@ public class ControlNodeActionsService {
     public Object sideloadStackexchange(Request request, Response response) throws Exception {
 
         Path sourcePath = Path.of(request.queryParams("source"));
-        if (!Files.exists(sourcePath)) {
-            Spark.halt(404);
-            return "No such file " + sourcePath;
-        }
-
         final int nodeId = Integer.parseInt(request.params("node"));
 
         eventLog.logEvent("USER-ACTION", "SIDELOAD STACKEXCHANGE " + nodeId);
@@ -127,5 +130,89 @@ public class ControlNodeActionsService {
         return "";
     }
 
+    private Object triggerAutoRecrawl(Request request, Response response) throws SQLException {
+        int nodeId = Integer.parseInt(request.params("id"));
+        var toCrawl = FileStorageId.parse(request.queryParams("source"));
+
+        changeActiveStorage(nodeId, FileStorageType.CRAWL_DATA, toCrawl);
+
+        executorClient.triggerRecrawl(
+                Context.fromRequest(request),
+                nodeId,
+                toCrawl
+        );
+
+        return "";
+    }
+
+    private Object triggerNewCrawl(Request request, Response response) throws SQLException {
+        int nodeId = Integer.parseInt(request.params("id"));
+
+        var toCrawl = FileStorageId.parse(request.queryParams("source"));
+
+        changeActiveStorage(nodeId, FileStorageType.CRAWL_SPEC, toCrawl);
+
+        executorClient.triggerCrawl(
+                Context.fromRequest(request),
+                nodeId,
+                toCrawl
+        );
+
+        return "";
+    }
+
+    private Object triggerAutoProcess(Request request, Response response) throws SQLException {
+        int nodeId = Integer.parseInt(request.params("id"));
+        var toProcess = FileStorageId.parse(request.queryParams("source"));
+
+        changeActiveStorage(nodeId, FileStorageType.PROCESSED_DATA, toProcess);
+
+        executorClient.triggerConvertAndLoad(Context.fromRequest(request),
+                nodeId,
+                toProcess);
+
+        return "";
+    }
+
+    private Object triggerLoadSelected(Request request, Response response) throws SQLException {
+        int nodeId = Integer.parseInt(request.params("id"));
+        String[] values = request.queryParamsValues("source");
+
+        List<FileStorageId> ids = Arrays.stream(values).map(FileStorageId::parse).toList();
+
+        changeActiveStorage(nodeId, FileStorageType.PROCESSED_DATA, ids.toArray(new FileStorageId[0]));
+
+        executorClient.loadProcessedData(Context.fromRequest(request),
+                nodeId,
+                new LoadParameters(ids)
+        );
+
+        return "";
+    }
+
+    private Object triggerRestoreBackup(Request request, Response response) {
+        int nodeId = Integer.parseInt(request.params("id"));
+
+        executorClient.restoreBackup(Context.fromRequest(request), nodeId, request.queryParams("source"));
+
+        return "";
+    }
+
+    /** Change the active storage for a node of a particular type. */
+    private void changeActiveStorage(int nodeId, FileStorageType type, FileStorageId... newActiveStorage) throws SQLException {
+        // It is desirable to have the active storage set to reflect which storage was last used
+        // for a particular node.
+
+        // Ideally we'd do this in a transaction, but as this is a reminder for the user, and not
+        // used for any actual processing, we don't need to be that strict.
+
+        for (var oldActiveStorage : fileStorageService.getActiveFileStorages(nodeId, type)) {
+            fileStorageService.setFileStorageState(oldActiveStorage, FileStorageState.UNSET);
+        }
+        for (var id : newActiveStorage) {
+            fileStorageService.setFileStorageState(id, FileStorageState.ACTIVE);
+        }
+    }
+
 
 }

code/services-core/control-service/src/main/java/nu/marginalia/control/node/svc/ControlNodeService.java

@@ -5,6 +5,7 @@ import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
 import nu.marginalia.client.Context;
 import nu.marginalia.client.ServiceMonitors;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.RedirectControl;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.node.model.*;
@@ -15,8 +16,6 @@ import nu.marginalia.nodecfg.NodeConfigurationService;
 import nu.marginalia.nodecfg.model.NodeConfiguration;
 import nu.marginalia.storage.FileStorageService;
 import nu.marginalia.executor.client.ExecutorClient;
-import nu.marginalia.executor.model.load.LoadParameters;
-import nu.marginalia.renderer.RendererFactory;
 import nu.marginalia.service.id.ServiceId;
 import nu.marginalia.storage.model.*;
 import org.slf4j.Logger;
@@ -32,7 +31,7 @@ import java.util.*;
 
 public class ControlNodeService {
     private final FileStorageService fileStorageService;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final EventLogService eventLogService;
     private final HeartbeatService heartbeatService;
     private final ExecutorClient executorClient;
@@ -46,7 +45,7 @@ public class ControlNodeService {
     @Inject
     public ControlNodeService(
             FileStorageService fileStorageService,
-            RendererFactory rendererFactory,
+            ControlRendererFactory rendererFactory,
             EventLogService eventLogService,
             HeartbeatService heartbeatService,
             ExecutorClient executorClient,
@@ -100,23 +99,9 @@ public class ControlNodeService {
         Spark.get("/public/nodes/:id/configuration", this::nodeConfigModel, configRenderer::render);
         Spark.post("/public/nodes/:id/configuration", this::updateConfigModel, configRenderer::render);
 
-        Spark.post("/public/nodes/:id/storage/recrawl-auto", this::triggerAutoRecrawl,
-                redirectControl.renderRedirectAcknowledgement("Recrawling", ".."));
-        Spark.post("/public/nodes/:id/storage/process-auto", this::triggerAutoProcess,
-                redirectControl.renderRedirectAcknowledgement("Processing", "..")
-                );
-        Spark.post("/public/nodes/:id/storage/load-selected", this::triggerLoadSelected,
-                redirectControl.renderRedirectAcknowledgement("Loading", "..")
-                );
-        Spark.post("/public/nodes/:id/storage/crawl/:fid", this::triggerCrawl,
-                redirectControl.renderRedirectAcknowledgement("Crawling", "..")
-                );
         Spark.post("/public/nodes/:id/storage/reset-state/:fid", this::resetState,
                 redirectControl.renderRedirectAcknowledgement("Restoring", "..")
         );
-        Spark.post("/public/nodes/:id/storage/backup-restore/:fid", this::triggerRestoreBackup,
-                redirectControl.renderRedirectAcknowledgement("Restoring", "..")
-                );
         Spark.post("/public/nodes/:id/actions/export-data", this::exportData,
                 redirectControl.renderRedirectAcknowledgement("Exporting", "../storage/exports")
                 );
@@ -159,18 +144,11 @@ public class ControlNodeService {
 
         int nextId = configs.stream().mapToInt(NodeConfiguration::node).map(i -> i+1).max().orElse(1);
 
-        return Map.of("nodes", nodeConfigurationService.getAll(),
+        return Map.of(
+                "nodes", nodeConfigurationService.getAll(),
                 "nextNodeId", nextId);
     }
 
-    private Object triggerCrawl(Request request, Response response) {
-        int nodeId = Integer.parseInt(request.params("id"));
-
-        executorClient.triggerCrawl(Context.fromRequest(request), nodeId, request.params("fid"));
-
-        return "";
-    }
-
     private Object stopProcess(Request request, Response response) {
         int nodeId = Integer.parseInt(request.params("id"));
         String processBase = request.params("processBase");
@@ -180,14 +158,6 @@ public class ControlNodeService {
         return "";
     }
 
-    private Object triggerRestoreBackup(Request request, Response response) {
-        int nodeId = Integer.parseInt(request.params("id"));
-
-        executorClient.restoreBackup(Context.fromRequest(request), nodeId, request.params("fid"));
-
-        return "";
-    }
-
     @SneakyThrows
     public String redirectToOverview(int nodeId) {
         return new Redirects.HtmlRedirect("/nodes/"+nodeId).render(null);
@@ -208,59 +178,21 @@ public class ControlNodeService {
         return "";
     }
 
-    private Object newSpecsModel(Request request, Response response) {
+    private Object newSpecsModel(Request request, Response response) throws SQLException {
         int nodeId = Integer.parseInt(request.params("id"));
 
         return Map.of(
+                "tab", Map.of("storage", true),
                 "node", new IndexNode(nodeId),
                 "view", Map.of("specs", true)
         );
     }
 
-    private Object triggerAutoRecrawl(Request request, Response response) throws SQLException {
+    private Object nodeActorsModel(Request request, Response response) throws SQLException {
-        int nodeId = Integer.parseInt(request.params("id"));
-
-        var toCrawl = fileStorageService.getOnlyActiveFileStorage(nodeId, FileStorageType.CRAWL_DATA);
-
-        executorClient.triggerRecrawl(
-                Context.fromRequest(request),
-                nodeId,
-                toCrawl.orElseThrow(AssertionError::new)
-        );
-
-        return "";
-    }
-
-    private Object triggerAutoProcess(Request request, Response response) throws SQLException {
-        int nodeId = Integer.parseInt(request.params("id"));
-
-        var toConvert = fileStorageService.getOnlyActiveFileStorage(nodeId, FileStorageType.CRAWL_DATA);
-
-        executorClient.triggerConvertAndLoad(Context.fromRequest(request),
-                nodeId,
-                toConvert.orElseThrow(AssertionError::new));
-
-        return "";
-    }
-
-    private Object triggerLoadSelected(Request request, Response response) throws SQLException {
-        int nodeId = Integer.parseInt(request.params("id"));
-
-        var toLoadStorages = fileStorageService.getActiveFileStorages(nodeId, FileStorageType.PROCESSED_DATA);
-
-        executorClient.loadProcessedData(Context.fromRequest(request),
-                nodeId,
-                new LoadParameters(toLoadStorages)
-        );
-
-        return "";
-    }
-
-
-    private Object nodeActorsModel(Request request, Response response) {
         int nodeId = Integer.parseInt(request.params("id"));
 
         return Map.of(
+                "tab", Map.of("actors", true),
                 "node", new IndexNode(nodeId),
                 "actors", executorClient.getActorStates(Context.fromRequest(request), nodeId).states()
         );
@@ -270,11 +202,18 @@ public class ControlNodeService {
         int nodeId = Integer.parseInt(request.params("id"));
 
         return Map.of(
+                "tab", Map.of("actions", true),
                 "node", new IndexNode(nodeId),
-                "currentCrawlData",
+                "view", Map.of(request.queryParams("view"), true),
-                        fileStorageService.getStorage(fileStorageService.getActiveFileStorages(nodeId, FileStorageType.CRAWL_DATA)),
+                "uploadDirContents", executorClient.listSideloadDir(Context.fromRequest(request), nodeId),
-                "currentProcessData",
+                "allBackups",
-                        fileStorageService.getStorage(fileStorageService.getActiveFileStorages(nodeId, FileStorageType.PROCESSED_DATA))
+                        fileStorageService.getEachFileStorage(nodeId, FileStorageType.BACKUP),
+                "allCrawlData",
+                        fileStorageService.getEachFileStorage(nodeId, FileStorageType.CRAWL_DATA),
+                "allProcessedData",
+                        fileStorageService.getEachFileStorage(nodeId, FileStorageType.PROCESSED_DATA),
+                "allCrawlSpecs",
+                        fileStorageService.getEachFileStorage(nodeId, FileStorageType.CRAWL_SPEC)
         );
     }
 
@@ -282,6 +221,7 @@ public class ControlNodeService {
         int nodeId = Integer.parseInt(request.params("id"));
 
         return Map.of(
+                "tab", Map.of("storage", true),
                 "view", Map.of("conf", true),
                 "node", new IndexNode(nodeId),
                 "storagebase", getStorageBaseList(nodeId)
@@ -303,6 +243,7 @@ public class ControlNodeService {
         };
 
         return Map.of(
+                "tab", Map.of("storage", true),
                 "view", Map.of(view, true),
                 "node", new IndexNode(nodeId),
                 "storage", makeFileStorageBaseWithStorage(getFileStorageIds(type, nodeId))
@@ -323,9 +264,11 @@ public class ControlNodeService {
         };
 
         return Map.of(
+                "tab", Map.of("storage", true),
                 "view", Map.of(view, true),
                 "node", new IndexNode(nodeId),
-                "storage", storage);
+                "storage", storage
+                );
     }
 
     private Object nodeConfigModel(Request request, Response response) throws SQLException {
@@ -340,6 +283,7 @@ public class ControlNodeService {
         }
 
         return Map.of(
+                "tab", Map.of("config", true),
                 "node", new IndexNode(nodeId),
                 "config", Objects.requireNonNull(nodeConfigurationService.get(nodeId), "Failed to fetch configuration"),
                 "storage", storage);
@@ -379,7 +323,8 @@ public class ControlNodeService {
                 "status", getStatus(config),
                 "events", getEvents(nodeId),
                 "processes", heartbeatService.getProcessHeartbeatsForNode(nodeId),
-                "jobs", heartbeatService.getTaskHeartbeatsForNode(nodeId)
+                "jobs", heartbeatService.getTaskHeartbeatsForNode(nodeId),
+                "tab", Map.of("overview", true)
                 );
     }
 

code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/ControlSysActionsService.java

@@ -3,6 +3,7 @@ package nu.marginalia.control.sys.svc;
 import com.google.inject.Inject;
 import lombok.SneakyThrows;
 import nu.marginalia.client.Context;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.actor.ControlActor;
 import nu.marginalia.control.actor.ControlActorService;
@@ -11,8 +12,6 @@ import nu.marginalia.executor.client.ExecutorClient;
 import nu.marginalia.mq.MessageQueueFactory;
 import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.nodecfg.NodeConfigurationService;
-import nu.marginalia.nodecfg.model.NodeConfiguration;
-import nu.marginalia.renderer.RendererFactory;
 import nu.marginalia.service.control.ServiceEventLog;
 import nu.marginalia.service.id.ServiceId;
 import nu.marginalia.storage.FileStorageService;
@@ -27,7 +26,7 @@ public class ControlSysActionsService {
     private final MqOutbox apiOutbox;
     private final DomainTypes domainTypes;
     private final ServiceEventLog eventLog;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final ControlActorService controlActorService;
     private final NodeConfigurationService nodeConfigurationService;
     private final FileStorageService fileStorageService;
@@ -37,7 +36,7 @@ public class ControlSysActionsService {
     public ControlSysActionsService(MessageQueueFactory mqFactory,
                                     DomainTypes domainTypes,
                                     ServiceEventLog eventLog,
-                                    RendererFactory rendererFactory,
+                                    ControlRendererFactory rendererFactory,
                                     ControlActorService controlActorService,
                                     NodeConfigurationService nodeConfigurationService,
                                     FileStorageService fileStorageService,

code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/DataSetsService.java

@@ -3,8 +3,8 @@ package nu.marginalia.control.sys.svc;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.db.DomainTypes;
-import nu.marginalia.renderer.RendererFactory;
 import spark.Request;
 import spark.Response;
 import spark.Spark;
@@ -17,12 +17,12 @@ import java.util.Map;
 public class DataSetsService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final DomainTypes domainTypes;
 
     @Inject
     public DataSetsService(HikariDataSource dataSource,
-                           RendererFactory rendererFactory,
+                           ControlRendererFactory rendererFactory,
                            DomainTypes domainTypes) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;

code/services-core/control-service/src/main/java/nu/marginalia/control/sys/svc/MessageQueueService.java

@@ -3,11 +3,11 @@ package nu.marginalia.control.sys.svc;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.ControlRendererFactory;
 import nu.marginalia.control.Redirects;
 import nu.marginalia.control.sys.model.MessageQueueEntry;
 import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.persistence.MqPersistence;
-import nu.marginalia.renderer.RendererFactory;
 import spark.Request;
 import spark.Response;
 import spark.Spark;
@@ -24,12 +24,12 @@ import java.util.Optional;
 public class MessageQueueService {
 
     private final HikariDataSource dataSource;
-    private final RendererFactory rendererFactory;
+    private final ControlRendererFactory rendererFactory;
     private final MqPersistence persistence;
 
     @Inject
     public MessageQueueService(HikariDataSource dataSource,
-                               RendererFactory rendererFactory,
+                               ControlRendererFactory rendererFactory,
                                MqPersistence persistence) {
         this.dataSource = dataSource;
         this.rendererFactory = rendererFactory;

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-export-data.hdb

@@ -0,0 +1,13 @@
+<h1 class="my-3">Export Data</h1>
+
+This will export database data: Domains, blacklist and domain links.
+
+<form method="post" action="actions/export-data" onsubmit="return confirm('Confirm export')">
+    <div class="my-3 py-3">
+        <div class="row">
+            <div class="col">
+                <button type="submit" class="btn btn-primary">Export</button>
+            </div>
+        </div>
+    </div>
+</form>

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-load.hdb

@@ -0,0 +1,38 @@
+<h1 class="my-3">Load</h1>
+
+{{#unless allProcessedData}}
+    No <a href="storage/processed">processed data</a> is currently set as active for this node.  Loading is not possible.
+{{/unless}}
+
+{{#if allProcessedData}}
+    <div class="my-3 p-3 border bg-light">
+    This will load <a href="storage/processed">processed data</a> on Node {{node.id}}.  This isn't typically necessary for crawled data,
+    but is needed for sideloaded data.  Select the processed data to load below.
+    </div>
+
+    <form method="post" action="actions/load">
+
+    <table class="table">
+        <tr>
+            <th>Use</th>
+            <th>Path</th>
+            <th>Description</th>
+            <th>Details</th>
+        </tr>
+
+        {{#each allProcessedData}}
+            <tr>
+                <td><input {{#if active}}checked{{/if}} class="form-check-input" type="checkbox" name="source" id="{{id}}" value="{{id}}"></td>
+                <td><label for="{{id}}" class="form-check-label" >{{path}}</label></td>
+                <td>{{description}}</td>
+                <td><a href="/nodes/{{node}}/storage/details?fid={{id}}">[Details]</a></td>
+            </tr>
+        {{/each}}
+    </table>
+        <button
+                class="btn btn-primary me-md-2"
+                onclick="return confirm('Confirm loading of the selected data on node {{node.id}}');"
+                type="submit">
+            Trigger Loading</button>
+    </form>
+{{/if}}

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-new-crawl.hdb

@@ -0,0 +1,53 @@
+<h1 class="my-3">New Crawl</h1>
+
+{{#unless allCrawlSpecs}}
+    No crawling specifications have been created.  You must create a crawl spec before you can crawl.
+    <a href="/nodes/{{node.id}}/storage/new-specs">create a crawl spec</a>.
+{{/unless}}
+
+{{#if allCrawlSpecs}}
+    <div class="my-3 p-3 border bg-light">
+    This will perform a new crawl on node {{node.id}} based on the crawl spec you select below.
+    Additional specifications can be created <a href="/nodes/{{node.id}}/storage/new-specs">with this form</a>.
+    </div>
+    <div class="my-3 p-3 border text-danger">
+        <p><strong>IMPORTANT!</strong>  Be sure you've read and understood the
+            <a href="https://github.com/MarginaliaSearch/MarginaliaSearch/blob/master/doc/crawling.md">crawling documentation</a>
+        before you begin a crawl.  You will be accessing real servers from your connection, and you may end up on IP greylists
+        that temporarily block your access to those servers for up to a few weeks; on rare occasions permanently.  The crawler
+        is generally polite and well-behaved, but it is still a crawler, and it will be accessing a lot of pages.
+    </p>
+    <p>
+        You run the risk of getting in trouble with your ISP, the law, your partner, your landlord, and/or your parents;
+        depending on terms of service, jurisdiction, marital status, living conditions and age.
+    <p>
+    </p>
+        This action is performed at your own risk.
+    </p>
+    </div>
+
+    <form method="post" action="actions/new-crawl">
+    <table class="table">
+        <tr>
+            <th>Use</th>
+            <th>Path</th>
+            <th>Description</th>
+            <th>Details</th>
+        </tr>
+
+        {{#each allCrawlSpecs}}
+            <tr>
+                <td><input {{#if active}}checked{{/if}} class="form-check-input" type="checkbox" name="source" id="{{id}}" value="{{id}}"></td>
+                <td><label for="{{id}}" class="form-check-label" >{{path}}</label></td>
+                <td>{{description}}</td>
+                <td><a href="/nodes/{{node}}/storage/details?fid={{id}}">[Details]</a></td>
+            </tr>
+        {{/each}}
+    </table>
+        <button
+                class="btn btn-primary me-md-2"
+                onclick="return confirm('Confirm crawling on node {{node.id}}');"
+                type="submit">
+            Trigger New  Crawl</button>
+    </form>
+{{/if}}

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-process.hdb

@@ -0,0 +1,39 @@
+<h1 class="my-3">Process Data</h1>
+
+<div class="my-3 p-3 border bg-light">
+    <p>
+        This will reprocess a set of crawl data on Node {{node.id}}.
+    </p>
+    <p>
+        The reprocessing will extract keywords, titles, and other metadata from the raw crawl data
+        and generate parquet files that can be loaded into the index.  This loading will be done automatically,
+        and unless the node is <a href="/nodes/{{node.id}}/configuration">configured</a>  to keep the processed data,
+        the processed data will be deleted after loading to save disk space.
+    </p>
+</div>
+
+<form method="post" action="actions/process">
+<p>Select the crawl data to use:</p>
+<table class="table">
+    <tr>
+        <th>Use</th>
+        <th>Path</th>
+        <th>Description</th>
+        <th>Details</th>
+    </tr>
+    {{#each allCrawlData}}
+        <tr>
+            <td><input {{#if active}}checked{{/if}} class="form-check-input" type="radio" name="source" id="{{id}}" value="{{id}}"></td>
+            <td><label for="{{id}}" class="form-check-label" >{{path}}</label></td>
+            <td>{{description}}</td>
+            <td><a href="/nodes/{{node}}/storage/details?fid={{id}}">[Details]</td>
+        </tr>
+    {{/each}}
+</table>
+
+    <button
+            class="btn btn-primary me-md-2"
+            onclick="return confirm('Confirm reprocessing of the selected data on node {{node.id}}');"
+            type="submit">
+        Trigger Reprocessing</button>
+</form>

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-recrawl.hdb

@@ -0,0 +1,48 @@
+<h1 class="my-3">Recrawl</h1>
+
+{{#unless allCrawlData}}
+    <p>
+    No <a href="storage/crawl">crawl data</a> currently exists on this node.  Recrawling is not possible.
+    First, create a crawl spec and do an initial crawl. Then, you can recrawl.
+    </p>
+{{/unless}}
+
+{{#if allCrawlData}}
+
+<div class="my-3 p-3 border bg-light">
+    <p>
+    This will perform a re-crawl on node {{node.id}}.
+    </p><p>
+    A re-crawl is a smart crawl that uses the existing crawl data
+    as a starting point.  It will use the existing crawl data to perform conditional GETs on the domains in the
+    crawl spec.  If the document has changed, it will be re-crawled.  If it has not changed, it will be skipped,
+    and the previous data will be retained.  This is both faster and easier on the target server.
+    </p>
+</div>
+
+<form method="post" action="actions/recrawl">
+<p>Select the crawl data to use:</p>
+<table class="table">
+    <tr>
+        <th>Use</th>
+        <th>Path</th>
+        <th>Description</th>
+        <th>Details</th>
+    </tr>
+{{#each allCrawlData}}
+    <tr>
+        <td><input {{#if active}}checked{{/if}} class="form-check-input" type="radio" name="source" id="{{id}}" value="{{id}}"></td>
+        <td><label for="{{id}}" class="form-check-label" >{{path}}</label></td>
+        <td>{{description}}</td>
+        <td><a href="/nodes/{{node}}/storage/details?fid={{id}}">[Details]</a></td>
+    </tr>
+{{/each}}
+</table>
+
+    <button
+            class="btn btn-primary me-md-2"
+            onclick="return confirm('Confirm recrawl of  the selected data on node {{node.id}}');"
+            type="submit">
+        Trigger Recrawl</button>
+</form>
+{{/if}}

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-repartition.hdb

@@ -0,0 +1,7 @@
+<h1 class="my-3">Repartition</h1>
+
+<p>This will recalculate the rankings and search sets for the index</p>
+
+<form method="post" action="actions/repartition-index" onsubmit="return confirm('Confirm repartition')">
+    <button type="submit" class="btn btn-primary">Trigger Repartitioning</button>
+</form>

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-restore-backup.hdb

@@ -0,0 +1,38 @@
+<h1 class="my-3">Load</h1>
+
+{{#unless allBackups}}
+    No <a href="storage/backups">processed data</a> exist on this node.  Backups are created when loading data into
+    the index.
+{{/unless}}
+
+{{#if allBackups}}
+    <div class="my-3 p-3 border bg-light">
+    This will restore an index backup on Node {{node.id}}.  The index will need to be re-generated after this operation,
+    this happens automatically.
+    </div>
+    <form method="post" action="actions/restore-backup">
+
+    <table class="table">
+        <tr>
+            <th>Use</th>
+            <th>Path</th>
+            <th>Description</th>
+            <th>Details</th>
+        </tr>
+
+        {{#each allBackups}}
+            <tr>
+                <td><input {{#if active}}checked{{/if}} class="form-check-input" type="radio" name="source" id="{{id}}" value="{{id}}"></td>
+                <td><label for="{{id}}" class="form-check-label" >{{path}}</label></td>
+                <td>{{description}}</td>
+                <td><a href="/nodes/{{node}}/storage/details?fid={{id}}">[Details]</a></td>
+            </tr>
+        {{/each}}
+    </table>
+        <button
+                class="btn btn-primary me-md-2"
+                onclick="return confirm('Confirm restoring selected data on node {{node.id}}');"
+                type="submit">
+            Trigger Restoration</button>
+    </form>
+{{/if}}

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-encyclopedia.hdb

@@ -0,0 +1,55 @@
+<h1 class="my-3">Sideload Encyclopedia</h1>
+
+<div class="my-3 p-3 border bg-light">
+<p>This will sideload a pre-converted MediaWiki-style OpenZim data set.
+    See the <a href="https://github.com/MarginaliaSearch/MarginaliaSearch/blob/master/doc/sideloading-howto.md">sideloading howto</a>
+    for instructions how to produce this file. </p>
+<p>Place an articles.db file in the upload directory on the server, and select it from the list
+    below.  </p>
+</div>
+<form method="post" action="actions/sideload-encyclopedia" onsubmit="return confirm('Confirm sideloading')">
+    <div class="my-3 py-3">
+
+        <table class="table">
+            <th></th><th>Filename</th><th>Size</th><th>Last Modified</th>
+            {{#each uploadDirContents.items}}
+                <tr>
+                    <td><input {{#if directory}}disabled{{/if}} class="form-check-input" type="radio" name="source" id="{{name}}" value="{{name}}"></td>
+                    <td {{#if directory}}class="text-muted"{{/if}}>
+                        <label class="form-check-label" for="inlineRadio1">{{name}}{{#if directory}}/{{/if}}</label>
+                    </td>
+                    <td>{{#unless directory}}{{size}}{{/unless}}</td>
+                    <td>{{lastModifiedTime}}</td>
+                </tr>
+            {{/each}}
+            {{#unless uploadDirContents.items}}
+                <tr>
+                    <td colspan="3">Nothing found in upload directory</td>
+                </tr>
+            {{/unless}}
+        </table>
+
+        <p>
+            <small class="text-muted">
+                The upload directory is typically mounted to /uploads on the server.  The external
+                directory is typically something like index-{{node.id}}/uploads.
+            </small>
+        </p>
+
+        <label for="baseUrl" class="form-label">Base URL</label>
+        <div class="col mb-3">
+            <input id="baseUrl" name="baseUrl" class="form-control" value="https://en.wikipedia.org/wiki/">
+        </div>
+
+        <small class="text-muted">
+            The base URL is used to construct the URL for each article.  The base URL is prepended to the article path,
+            and the result is used as the URL for the article.  The default value will point to English Wikipedia.
+        </small>
+
+        <div class="row mt-3">
+            <div class="col">
+                <button {{#unless uploadDirContents.items}}disabled{{/unless}} type="submit" class="btn btn-primary">Sideload Encyclopedia</button>
+            </div>
+        </div>
+    </div>
+</form>

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-stackexchange.hdb

@@ -0,0 +1,41 @@
+<h1 class="my-3">Sideload Stackexchange</h1>
+
+<div class="my-3 p-3 border bg-light">
+This will load a set of pre-converted stackexchange .db files
+<p>
+See the <a href="https://github.com/MarginaliaSearch/MarginaliaSearch/blob/master/doc/sideloading-howto.md">sideloading howto</a>
+for instructions how to produce this file. </p>
+</div>
+
+<form method="post" action="actions/sideload-stackexchange" onsubmit="return confirm('Confirm sideloading')">
+    <div class="my-3 py-3">
+
+        <table class="table">
+            <th></th><th>Filename</th><th>Size</th><th>Last Modified</th>
+            {{#each uploadDirContents.items}}
+                <tr>
+                    <td><input {{#unless directory}}disabled{{/unless}} class="form-check-input" type="radio" name="source" id="{{name}}" value="{{name}}"></td>
+                    <td {{#unless directory}}class="text-muted"{{/unless}}>
+                        <label class="form-check-label" for="inlineRadio1">{{name}}{{#if directory}}/{{/if}}</label>
+                    </td>
+                    <td>{{#unless directory}}{{size}}{{/unless}}</td>
+                    <td>{{lastModifiedTime}}</td>
+                </tr>
+            {{/each}}
+            {{#unless uploadDirContents.items}}
+                <tr>
+                    <td colspan="3">Nothing found in upload directory</td>
+                </tr>
+            {{/unless}}
+        </table>
+
+        <p>
+            <small class="text-muted">
+                The upload directory is typically mounted to /uploads on the server.  The external
+                directory is typically something like index-{{node.id}}/uploads.
+            </small>
+        </p>
+
+        <button type="submit" {{#unless uploadDirContents.items}}disabled{{/unless}} class="btn btn-primary">Sideload Stackexchange</button>
+    </div>
+</form>

code/services-core/control-service/src/main/resources/templates/control/node/actions/partial-sideload-warc.hdb

@@ -0,0 +1,39 @@
+<h1 class="my-3">Sideload WARC</h1>
+
+<div class="my-3 p-3 border bg-light">
+This will load HTML from a directory structure containing WARC files, one per domain.
+A warc export can be created using e.g. wget: <p>
+<code>$ wget -r --warc-file=www.marginalia.nu www.marginalia.nu</code>
+</div>
+<form method="post" action="actions/sideload-warc" onsubmit="return confirm('Confirm sideloading')">
+    <div class="my-3 py-3">
+
+        <table class="table">
+            <th></th><th>Filename</th><th>Size</th><th>Last Modified</th>
+            {{#each uploadDirContents.items}}
+                <tr>
+                    <td><input {{#unless directory}}disabled{{/unless}} class="form-check-input" type="radio" name="source" id="{{name}}" value="{{name}}"></td>
+                    <td {{#unless directory}}class="text-muted"{{/unless}}>
+                        <label class="form-check-label" for="inlineRadio1">{{name}}{{#if directory}}/{{/if}}</label>
+                    </td>
+                    <td>{{#unless directory}}{{size}}{{/unless}}</td>
+                    <td>{{lastModifiedTime}}</td>
+                </tr>
+            {{/each}}
+            {{#unless uploadDirContents.items}}
+                <tr>
+                    <td colspan="3">Nothing found in upload directory</td>
+                </tr>
+            {{/unless}}
+        </table>
+
+        <p>
+            <small class="text-muted">
+                The upload directory is typically mounted to /uploads on the server.  The external
+                directory is typically something like index-{{node.id}}/uploads.
+            </small>
+        </p>
+
+        <button type="submit" {{#unless uploadDirContents.items}}disabled{{/unless}} class="btn btn-primary">Sideload WARCs</button>
+    </div>
+</form>

code/services-core/control-service/src/main/resources/templates/control/node/node-actions.hdb

@@ -7,352 +7,20 @@
 
 <div class="container">
 
-    <nav aria-label="breadcrumb">
+    {{> control/node/partial-node-nav }}
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item"><a href="/nodes">nodes</a></li>
-            <li class="breadcrumb-item">node-{{node.id}}</li>
-        </ol>
-    </nav>
-
-    <nav class="nav nav-tabs">
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link active" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
-
-    <div class="accordion mt-1" id="accordionActions">
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseTriggerCrawl"
-                        aria-expanded="false"
-                        aria-controls="collapseTriggerCrawl">
-                    Trigger Crawl
-                </button>
-            </h2>
-            <div id="collapseTriggerCrawl" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-
-                {{#unless currentCrawlData}}
-                    No <a href="storage/crawl">crawl data</a> is currently set as active for this node.  A crawl is not possible.
-                {{/unless}}
-
-                {{#if currentCrawlData}}
-                    This will trigger a crawl on Node {{node.id}}.  This <a href="storage/crawl">crawl data</a> will be loaded:
-                <table class="table">
-                    <tr>
-                        <th>Date</th>
-                        <th>Path</th>
-                        <th>Description</th>
-                    </tr>
-                    {{#each currentCrawlData}}
-                    <tr>
-                        <td>{{date}}</td>
-                        <td><a href="storage/details?fid={{id}}">{{path}}</a></td>
-                        <td>{{description}}</td>
-                    </tr>
-                    {{/each}}
-                </table>
-                    <form method="post" action="storage/recrawl-auto">
-                        <button
-                                class="btn btn-primary me-md-2"
-                                onclick="return confirm('Confirm recrawl of the current ACTIVE crawl set on node {{node.id}}');"
-                                type="submit">
-                            Trigger Recrawl</button>
-                    </form>
-                {{/if}}
 
+    {{#if view.new-crawl}} {{> control/node/actions/partial-new-crawl }} {{/if}}
+    {{#if view.re-crawl}} {{> control/node/actions/partial-recrawl }} {{/if}}
+    {{#if view.process}} {{> control/node/actions/partial-process }} {{/if}}
+    {{#if view.load}} {{> control/node/actions/partial-load }} {{/if}}
+    {{#if view.repartition}} {{> control/node/actions/partial-repartition }} {{/if}}
+    {{#if view.sideload-encyclopedia}} {{> control/node/actions/partial-sideload-encyclopedia }} {{/if}}
+    {{#if view.sideload-stackexchange}} {{> control/node/actions/partial-sideload-stackexchange }} {{/if}}
+    {{#if view.sideload-warc}} {{> control/node/actions/partial-sideload-warc }} {{/if}}
+    {{#if view.export-data}} {{> control/node/actions/partial-export-data }} {{/if}}
+    {{#if view.restore-backup}} {{> control/node/actions/partial-restore-backup }} {{/if}}
+    <div class="mt-10">&nbsp;</div>
 </div>
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseProcessCrawlData"
-                        aria-expanded="false"
-                        aria-controls="collapseProcessCrawlData">
-                    Process Crawl Data
-                </button>
-            </h2>
-            <div id="collapseProcessCrawlData" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-
-                {{#unless currentCrawlData}}
-                    No <a href="storage/crawl">crawl data</a> is currently set as active for this node.  Processing is not possible.
-                {{/unless}}
-
-                {{#if currentCrawlData}}
-                    This will reprocess crawl data on Node {{node.id}}.  This <a href="storage/crawl">crawl data</a> will be loaded:
-                    <table class="table">
-                        <tr>
-                            <th>Date</th>
-                            <th>Path</th>
-                            <th>Description</th>
-                        </tr>
-
-                        {{#each currentCrawlData}}
-                        <tr>
-                            <td>{{date}}</td>
-                            <td><a href="storage/details?fid={{id}}">{{path}}</a></td>
-                            <td>{{description}}</td>
-                        </tr>
-                        {{/each}}
-                    </table>
-                    <form method="post" action="storage/process-auto">
-                        <button
-                                class="btn btn-primary me-md-2"
-                                onclick="return confirm('Confirm reprocessing of the current ACTIVE data on node {{node.id}}');"
-                                type="submit">
-                            Trigger Reprocessing</button>
-                    </form>
-                {{/if}}
-            </div>
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseLoadData"
-                        aria-expanded="false"
-                        aria-controls="collapseLoadData">
-                    Load Processed Data
-                </button>
-            </h2>
-            <div id="collapseLoadData" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-
-                {{#unless currentProcessData}}
-                    No <a href="storage/processed">processed data</a> is currently set as active for this node.  Loading is not possible.
-                {{/unless}}
-
-                {{#if currentProcessData}}
-                    This will load processed data on Node {{node.id}}.  This <a href="storage/processed">processed data</a> will be loaded:
-                    <table class="table">
-                        <tr>
-                            <th>Date</th>
-                            <th>Path</th>
-                            <th>Description</th>
-                        </tr>
-
-                        {{#each currentProcessData}}
-                            <tr>
-                                <td>{{date}}</td>
-                                <td><a href="storage/details?fid={{id}}">{{path}}</a></td>
-                                <td>{{description}}</td>
-                            </tr>
-                        {{/each}}
-                    </table>
-                    <form method="post" action="storage/load-selected">
-                        <button
-                                class="btn btn-primary me-md-2"
-                                onclick="return confirm('Confirm loading of the current ACTIVE data on node {{node.id}}');"
-                                type="submit">
-                            Trigger Loading</button>
-                    </form>
-                {{/if}}
-            </div>
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseRepartition"
-                        aria-expanded="true"
-                        aria-controls="collapseRepartition">
-                    Repartition Index
-                </button>
-            </h2>
-            <div id="collapseRepartition" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-                <p>This will recalculate the rankings and search sets for the index</p>
-
-                <form method="post" action="actions/repartition-index" onsubmit="return confirm('Confirm repartition')">
-                    <button type="submit" class="btn btn-primary">Trigger Repartitioning</button>
-                </form>
-            </div>
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseSideloadEncyclopedia"
-                        aria-expanded="false"
-                        aria-controls="collapseSideloadEncyclopedia">
-                    Sideload Encyclopedia
-                </button>
-            </h2>
-            <div id="collapseSideloadEncyclopedia" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-
-                This will load pre-digested encyclopedia data from an <tt>encyclopedia.marginalia.nu</tt>-style database.
-
-            <form method="post" action="actions/sideload-encyclopedia" onsubmit="return confirm('Confirm sideloading')">
-                <div class="my-3 py-3">
-
-                    <label for="baseUrl" class="form-label">Base URL</label>
-                    <div class="col mb-3">
-                        <input id="baseUrl" name="baseUrl" class="form-control" value="https://en.wikipedia.org/wiki/">
-                    </div>
-
-                    <label for="source" class="form-label">Path to articles.db accessible from the node on the server</label>
-
-                    <div class="row">
-                        <div class="col">
-                            <input id="source" name="source" class="form-control" value="">
-                        </div>
-                        <div class="col">
-                            <button type="submit" class="btn btn-primary">Sideload Encyclopedia</button>
-                        </div>
-                    </div>
-                </div>
-            </form>
-            </div>
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseSideloadStackexchange"
-                        aria-expanded="false"
-                        aria-controls="collapseSideloadStackexchange">
-                    Sideload Stackexchange
-                </button>
-            </h2>
-            <div id="collapseSideloadStackexchange" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-                This will load a set of pre-converted stackexchange .db files
-
-                <form method="post" action="actions/sideload-stackexchange" onsubmit="return confirm('Confirm sideloading')">
-                    <div class="my-3 py-3">
-                        <label for="source" class="form-label">Directory with .db files accessible from the node on the server</label>
-
-                        <div class="row">
-                            <div class="col">
-                                <input id="source" name="source" class="form-control" value="">
-                            </div>
-                            <div class="col">
-                                <button type="submit" class="btn btn-primary">Sideload Stackexchange</button>
-                            </div>
-                        </div>
-                    </div>
-                </form>
-            </div>
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseSideloadDirtree"
-                        aria-expanded="false"
-                        aria-controls="collapseSideloadDirtree">
-                    Sideload Dirtree
-                </button>
-            </h2>
-
-            <div id="collapseSideloadDirtree" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-                This will load HTML from a directory structure as specified by a YAML file.
-
-                <form method="post" action="actions/sideload-dirtree" onsubmit="return confirm('Confirm sideloading')">
-                    <div class="my-3 py-3">
-                        <label for="source" class="form-label">Path yaml file location on server</label>
-
-                        <div class="row">
-                            <div class="col">
-                                <input id="source" name="source" class="form-control" value="">
-                            </div>
-                            <div class="col">
-                                <button type="submit" class="btn btn-primary">Sideload Dirtree</button>
-                            </div>
-                        </div>
-                    </div>
-                </form>
-            </div>
-
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseSideloadWarc"
-                        aria-expanded="false"
-                        aria-controls="collapseSideloadWarc">
-                    Sideload WARC Files
-                </button>
-            </h2>
-
-            <div id="collapseSideloadWarc" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-                This will load HTML from a directory structure containing WARC files, one per domain.
-
-                <form method="post" action="actions/sideload-warc" onsubmit="return confirm('Confirm sideloading')">
-                    <div class="my-3 py-3">
-                        <label for="source" class="form-label">Path WARC file location on server</label>
-
-                        <div class="row">
-                            <div class="col">
-                                <input id="source" name="source" class="form-control" value="">
-                            </div>
-                            <div class="col">
-                                <button type="submit" class="btn btn-primary">Sideload WARC Files</button>
-                            </div>
-                        </div>
-                    </div>
-                </form>
-            </div>
-
-        </div>
-
-        <div class="accordion-item">
-            <h2 class="accordion-header">
-                <button class="accordion-button collapsed"
-                        type="button"
-                        data-bs-toggle="collapse"
-                        data-bs-target="#collapseExportData"
-                        aria-expanded="false"
-                        aria-controls="collapseExportData">
-                    Export Database Data
-                </button>
-            </h2>
-
-            <div id="collapseExportData" class="accordion-collapse collapse p-3" data-bs-parent="#accordionActions">
-                This will export database data: Domains, blacklist and domain links.
-
-                <form method="post" action="actions/export-data" onsubmit="return confirm('Confirm export')">
-                    <div class="my-3 py-3">
-                        <div class="row">
-                            <div class="col">
-                                <button type="submit" class="btn btn-primary">Export</button>
-                            </div>
-                        </div>
-                    </div>
-                </form>
-            </div>
-        </div>
-    </div>
-</div>
-
 </body>
 {{> control/partials/foot-includes }}
 </html>

code/services-core/control-service/src/main/resources/templates/control/node/node-actors.hdb

@@ -7,31 +7,7 @@
 
 <div class="container">
 
-    <nav aria-label="breadcrumb">
+    {{> control/node/partial-node-nav }}
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item"><a href="/nodes">nodes</a></li>
-            <li class="breadcrumb-item">node-{{node.id}}</li>
-        </ol>
-    </nav>
-
-    <nav class="nav nav-tabs">
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link active" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
 
     <div class="mt-2">
         {{> control/partials/actors-table }}

code/services-core/control-service/src/main/resources/templates/control/node/node-config.hdb

@@ -7,31 +7,7 @@
 
 <div class="container">
 
-    <nav aria-label="breadcrumb">
+    {{> control/node/partial-node-nav }}
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item"><a href="/nodes">nodes</a></li>
-            <li class="breadcrumb-item">node-{{node.id}}</li>
-        </ol>
-    </nav>
-
-    <nav class="nav nav-tabs">
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link active" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
 
     <h1 class="my-5">Node Configuration</h1>
 

code/services-core/control-service/src/main/resources/templates/control/node/node-new-specs-form.hdb

@@ -7,39 +7,21 @@
 
 <div class="container">
 
-    <nav aria-label="breadcrumb">
+    {{> control/node/partial-node-nav }}
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item"><a href="/nodes">nodes</a></li>
-            <li class="breadcrumb-item">node-{{node.id}}</li>
-        </ol>
-    </nav>
-
-    <nav class="nav nav-tabs">
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link active" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
 
     <div class="container">
         {{>control/partials/storage-types}}
 
-        <div class="col-4">
+        <div>
             <h2>Create New Specification</h2>
 
-            <p>To create a new specification fill out the form below. </p>
+            <div class="my-3 p-3 border bg-light">
+            <p>A crawling specification is a list of domain names to crawl.</p>
+            <p>
+                If you are just looking to test the software, feel free to use <a href="https://downloads.marginalia.nu/domain-list-test.txt">this
+                short list of marginalia-related websites</a>, that are safe to crawl repeatedly without causing any problems.
+            </p>
+            </div>
 
             <form method="post" action="new-specs">
                 <div class="form">

code/services-core/control-service/src/main/resources/templates/control/node/node-overview.hdb

@@ -7,31 +7,7 @@
 
 <div class="container">
 
-    <nav aria-label="breadcrumb">
+    {{> control/node/partial-node-nav }}
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item"><a href="/nodes">nodes</a></li>
-            <li class="breadcrumb-item">node-{{node.id}}</li>
-        </ol>
-    </nav>
-
-    <nav class="nav nav-tabs">
-        <li class="nav-item">
-            <a class="nav-link active" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
 
     <div class="mt-2">
     {{> control/partials/processes-table }}

code/services-core/control-service/src/main/resources/templates/control/node/node-storage-conf.hdb

@@ -7,31 +7,7 @@
 
 <div class="container">
 
-    <nav aria-label="breadcrumb">
+    {{> control/node/partial-node-nav }}
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item"><a href="/nodes">nodes</a></li>
-            <li class="breadcrumb-item">node-{{node.id}}</li>
-        </ol>
-    </nav>
-
-    <nav class="nav nav-tabs">
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link active" aria-current="page" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
 
     <div class="container">
         {{>control/partials/storage-types}}

code/services-core/control-service/src/main/resources/templates/control/node/node-storage-details.hdb

@@ -14,24 +14,7 @@
         </ol>
     </nav>
 
-    <nav class="nav nav-tabs">
+    {{> control/node/partial-node-nav }}
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link active" aria-current="page" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
 
     <div class="container">
         {{>control/partials/storage-types}}
@@ -59,14 +42,6 @@
                 <th>Trigger</th>
             </tr>
             {{#with storage.self}}
-                {{#if isCrawlable}}
-                    <form method="post" action="/nodes/{{node.id}}/storage/crawl/{{storage.id}}" onsubmit="return confirm('Confirm crawling of {{storage.path}}')">
-                        <tr>
-                            <td>Perform a full crawl of this data into a new file storage area</td>
-                            <td><button class="btn btn-primary" type="submit">Crawl</button></td>
-                        </tr>
-                    </form>
-                {{/if}}
                 {{#if isStatusNew}}
                     <form method="post" action="/nodes/{{node.id}}/storage/reset-state/{{storage.id}}" onsubmit="return confirm('Confirm removal of \'NEW\' state for {{storage.path}}')">
                         <tr>
@@ -75,6 +50,7 @@
                         </tr>
                     </form>
                 {{/if}}
+
                 {{#if isAtagsExportable}}
                     <form method="post" action="/nodes/{{node.id}}/storage/{{storage.id}}/export-atags" onsubmit="return confirm('Confirm export of anchor tags from {{storage.path}}')">
                         <tr>
@@ -83,14 +59,6 @@
                         </tr>
                     </form>
                 {{/if}}
-                {{#if isRestorable}}
-                    <form method="post" action="/nodes/{{node.id}}/storage/backup-restore/{{storage.id}}" onsubmit="return confirm('Confirm restoring backup {{storage.path}}')">
-                        <tr>
-                            <td>Restore into live index</td>
-                            <td><button class="btn btn-primary"  type="submit">Restore</button></td>
-                        </tr>
-                    </form>
-                {{/if}}
 
                 {{#if isDeletable}}
                     <form method="post" action="/nodes/{{node.id}}/storage/{{storage.id}}/delete" onsubmit="return confirm('Confirm deletion of {{storage.path}}')">

code/services-core/control-service/src/main/resources/templates/control/node/node-storage-list.hdb

@@ -8,31 +8,7 @@
 
 <div class="container">
 
-    <nav aria-label="breadcrumb">
+    {{> control/node/partial-node-nav }}
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item"><a href="/nodes">nodes</a></li>
-            <li class="breadcrumb-item">node-{{node.id}}</li>
-        </ol>
-    </nav>
-
-    <nav class="nav nav-tabs">
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/">Overview</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actions">Actions</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/actors">Actors</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link active" aria-current="page" href="/nodes/{{node.id}}/storage/">Storage</a>
-        </li>
-        <li class="nav-item">
-            <a class="nav-link" href="/nodes/{{node.id}}/configuration">Configuration</a>
-        </li>
-
-    </nav>
 
     <div class="container">
         {{>control/partials/storage-types}}
@@ -72,14 +48,14 @@
                             {{#if storage.noState}}
                                 <div class="form-check form-switch">
                                     <label class="form-check-label" for="flexSwitchCheckDefault">Archived</label>
-                                    <input class="form-check-input" type="checkbox" id="flexSwitchCheckDefault"  onclick="setActive({{storage.id}})">
+                                    <input class="form-check-input" type="checkbox" id="flexSwitchCheckDefault">
                                 </div>
                             {{/if}}
 
                             {{#if storage.active}}
                                 <div class="form-check form-switch">
                                     <label class="form-check-label" for="flexSwitchCheckDefault">Active</label>
-                                    <input class="form-check-input" type="checkbox" id="flexSwitchCheckDefault" checked onclick="setArchived({{storage.id}})">
+                                    <input class="form-check-input" type="checkbox" id="flexSwitchCheckDefault">
                                 </div>
                             {{/if}}
                         </td>
@@ -96,79 +72,6 @@
                 </tr>
             {{/unless}}
         </table>
-        <script language="javascript">
-            function setActive(fileId) {
-                var xhr = new XMLHttpRequest();
-                xhr.open("POST", "/nodes/{{node.id}}/storage/"+fileId+"/enable");
-        xhr.onload = () => {
-            window.location.href=window.location.href;
-        }
-        xhr.send();
-        }
-        function setArchived(fileId) {
-            var xhr = new XMLHttpRequest();
-            xhr.open("POST", "/nodes/{{node.id}}/storage/"+fileId+"/disable");
-            xhr.onload = () => {
-                window.location.href=window.location.href;
-            }
-            xhr.send();
-        }
-        </script>
-
-        {{#if view.specs}}
-            <div class="m-3 p-3 border bg-light">
-            <h3 class="my-3">About Specs</h3>
-            <p>Crawl specifications are set of domains to be crawled.  Normally the crawler uses the domain database
-               to find domains to crawl, but sometimes, when bootstrapping a system or during testing, it's useful
-               to be able to specify which domains to crawl with more control.
-            </p>
-            <p>The <em>ACTIVE</em> status does nothing for crawl specs.</p>
-            <p><a href="new-specs">Create New Crawl Spec</p>
-            </div>
-        {{/if}}
-        {{#if view.crawl}}
-            <div class="m-3 p-3 border bg-light">
-                <h3 class="my-3">About Crawls</h3>
-                <p>Crawl data is the downloaded and compressed raw HTML of a website.</p>
-                <p>A new crawl may be bootstrappped from a Crawl Set</p>
-                <p>Unlike other data sets, only a single crawl set may be <em>ACTIVE</em> at any given time.</p>
-                <h4 class="my-3">Recrawl</h4>
-                <p>A set of existing crawl data can be <em>recrawled</em> and amended with new domains
-                    and a fresh look at the existing domains.  This is an operation that uses the currently <em>ACTIVE</em>
-                    crawl data.
-                </p>
-                <h4 class="my-3">Processing</h4>
-                <p>
-                    Before the data can be loaded into the search engine, it needs to be processed.  The process operation
-                    uses the currently <em>ACTIVE</em> crawl set.
-                </p>
-            </div>
-        {{/if}}
-        {{#if view.processed}}
-        <div class="m-3 p-3 border bg-light">
-            <h3 class="my-3">Processed Data</h3>
-            <p>
-            Processed data is HTML that has been analyzed and had its properties and keywords extracted. If one or more
-            processed data sets is <em>ACTIVE</em>, it can be Loaded into the system to make it searchable.  Loading processed
-            data also creates a backup of the Index Journal and Link Database that can be quickly restored in the event
-            of a problem.
-            </p>
-            <p>
-                Processed data may come from crawl data, but it may also be sideloaded into the system from e.g.
-                stackexchange data dumps through the Node Actions view.
-            </p>
-        </div>
-        {{/if}}
-        {{#if view.backup}}
-        <div class="m-3 p-3 border bg-light">
-            <h3 class="my-3">Backups</h3>
-            <p>
-                Backups are snapshots of the Index Journal and Link Database immediately before a searchable index
-                is constructed.  They are relatively quick to restore and allow you to roll back a problematic deployment
-                or recover from disk failures (assuming they are on a different drive).
-            </p>
-        </div>
-        {{/if}}
             </div>
     </div>
 </body>

code/services-core/control-service/src/main/resources/templates/control/node/partial-node-nav.hdb

@@ -0,0 +1,33 @@
+
+<h1 class="my-3">Index Node {{node.id}}</h1>
+<nav class="nav nav-tabs">
+    <li class="nav-item">
+        <a class="nav-link {{#if tab.overview}}active{{/if}}" href="/nodes/{{node.id}}/">Overview</a>
+    </li>
+    <li class="nav-item dropdown">
+        <a class="nav-link dropdown-toggle {{#if tab.actions}}active{{/if}}" data-bs-toggle="dropdown" href="#" role="button" aria-expanded="false">Actions</a>
+        <ul class="dropdown-menu">
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=new-crawl">New Crawl</a></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=re-crawl">Re-crawl</a></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=process">Process Crawl Data</a></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=load">Load Crawl Data</a></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=repartition">Repartition Index</a></li>
+            <li><hr class="dropdown-divider"></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=sideload-encyclopedia">Sideload Encyclopedia</a></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=sideload-stackexchange">Sideload Stackexchange</a></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=sideload-warc">Sideload WARC Files</a></li>
+            <li><hr class="dropdown-divider"></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=export-data">Export Database Data</a></li>
+            <li><a class="dropdown-item" href="/nodes/{{node.id}}/actions?view=restore-backup">Restore Index Backup</a></li>
+        </ul>
+    </li>
+    <li class="nav-item">
+        <a class="nav-link {{#if tab.actors}}active{{/if}}" href="/nodes/{{node.id}}/actors">Actors</a>
+    </li>
+    <li class="nav-item">
+        <a class="nav-link {{#if tab.storage}}active{{/if}}" href="/nodes/{{node.id}}/storage/">Storage</a>
+    </li>
+    <li class="nav-item">
+        <a class="nav-link {{#if tab.config}}active{{/if}}" href="/nodes/{{node.id}}/configuration">Configuration</a>
+    </li>
+</nav>

code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb

@@ -18,11 +18,20 @@
 						<li><a class="dropdown-item" href="/review-random-domains" title="Review random domains list">Random Exploration</a></li>
 					</ul>
 				</li>
+                <li class="nav-item dropdown">
+                    <a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button"  aria-expanded="false">Index Nodes</a>
+                    <ul class="dropdown-menu">
+                        <li><a class="dropdown-item" href="/nodes" title="Node Overview">Overview</a></li>
+                        <li><hr class="dropdown-divider"></li>
+                        {{#each global-context.nodes}}
+                            <li><a class="dropdown-item" href="/nodes/{{node}}" title="Manage Node {{node}}">Node {{node}}</a></li>
+                        {{/each}}
+                    </ul>
+                </li>
 				<li class="nav-item dropdown">
 					<a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button"  aria-expanded="false">System</a>
 					<ul class="dropdown-menu">
                         <li><a class="dropdown-item" href="/actions" title="System actions">Actions</a></li>
-                        <li><a class="dropdown-item" href="/nodes" title="View and configure index nodes">Nodes</a></li>
                         <li><a class="dropdown-item" href="/datasets" title="View and update the data sets">Datasets</a></li>
                         <li><a class="dropdown-item" href="/events" title="View the event log">Events</a></li>
 						<li><a class="dropdown-item" href="/message-queue" title="View or manipulate the system message queue">Message Queue</a></li>

code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorSvc.java

@@ -65,6 +65,7 @@ public class ExecutorSvc extends Service {
         Spark.post("/process/load", processingService::startLoad);
         Spark.post("/process/adjacency-calculation", processingService::startAdjacencyCalculation);
 
+        Spark.get("/sideload/", sideloadService::listUploadDir, gson::toJson);
         Spark.post("/sideload/dirtree", sideloadService::sideloadDirtree);
         Spark.post("/sideload/warc", sideloadService::sideloadWarc);
         Spark.post("/sideload/stackexchange", sideloadService::sideloadStackexchange);

code/services-core/executor-service/src/main/java/nu/marginalia/executor/svc/SideloadService.java

@@ -1,12 +1,21 @@
 package nu.marginalia.executor.svc;
 
 import com.google.inject.Inject;
+import nu.marginalia.WmsaHome;
 import nu.marginalia.actor.ExecutorActor;
 import nu.marginalia.actor.ExecutorActorControlService;
 import nu.marginalia.actor.task.ConvertActor;
+import nu.marginalia.executor.upload.UploadDirContents;
+import nu.marginalia.executor.upload.UploadDirItem;
 import spark.Request;
 import spark.Response;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
 public class SideloadService {
     private final ExecutorActorControlService actorControlService;
 
@@ -36,4 +45,15 @@ public class SideloadService {
         actorControlService.startFrom(ExecutorActor.CONVERT, new ConvertActor.ConvertStackexchange(request.queryParams("path")));
         return "";
     }
+
+    public UploadDirContents listUploadDir(Request request, Response response) throws IOException {
+        Path uploadDir = WmsaHome.getUploadDir();
+
+        try (var items = Files.list(uploadDir)) {
+            return new UploadDirContents(uploadDir.toString(),
+                    items.map(UploadDirItem::fromPath).toList());
+        }
+
+    }
+
 }

code/services-core/executor-service/src/test/java/nu/marginalia/executor/ExecutorSvcApiIntegrationTest.java

@@ -80,7 +80,7 @@ public class ExecutorSvcApiIntegrationTest {
 
     @Test
     public void triggerCrawl() throws Exception {
-        testInstances.client.triggerCrawl(Context.internal(), 0, "1");
+        testInstances.client.triggerCrawl(Context.internal(), 0, FileStorageId.of(1));
 
         Mockito.verify(testInstances.actorControlService).startFrom(eq(ExecutorActor.CRAWL), any());
     }

docker-compose-barebones.yml

@@ -35,6 +35,7 @@ x-p1: &partition-1
     - work-1:/work
     - backup-1:/backup
     - samples-1:/storage
+    - uploads-1:/uploads
   networks:
     - wmsa
   depends_on:
@@ -172,3 +173,9 @@ volumes:
       type: none
       o: bind
       device: run/node-1/backup
+  uploads-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-1/uploads

docker-compose.yml

@@ -29,6 +29,7 @@ x-p1: &partition-1
     - work-1:/work
     - backup-1:/backup
     - samples-1:/storage
+    - uploads-1:/uploads
   networks:
     - wmsa
   depends_on:
@@ -47,6 +48,7 @@ x-p2: &partition-2
     - work-2:/work
     - backup-2:/backup
     - samples-2:/storage
+    - uploads-2:/uploads
   networks:
     - wmsa
   depends_on:
@@ -275,6 +277,12 @@ volumes:
       type: none
       o: bind
       device: run/node-1/backup
+  uploads-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-1/uploads
   samples-2:
     driver: local
     driver_opts:
@@ -299,3 +307,9 @@ volumes:
       type: none
       o: bind
       device: run/node-2/backup
+  uploads-2:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: run/node-2/uploads

run/install/docker-compose.yml.template

@@ -31,7 +31,8 @@ x-p1: &partition-1
     - index-1:/idx
     - work-1:/work
     - backup-1:/backup
-    - samples-1:/storage
+    - storage-1:/storage
+    - uploads-1:/uploads
   networks:
     - wmsa
   environment:
@@ -200,12 +201,18 @@ volumes:
       type: none
       o: bind
       device: ${INSTALL_DIR}/data
-  samples-1:
+  storage-1:
     driver: local
     driver_opts:
       type: none
       o: bind
       device: ${INSTALL_DIR}/index-1/storage
+  uploads-1:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${INSTALL_DIR}/index-1/uploads
   index-1:
     driver: local
     driver_opts:

run/setup.sh

@@ -20,7 +20,7 @@ function download_model {
 pushd $(dirname $0)
 
 mkdir -p model logs db  install  data samples
-mkdir -p {node-1,node-2}/{work,index,backup,samples/export}
+mkdir -p {node-1,node-2}/{work,index,backup,samples/export,uploads}
 
 download_model model/English.DICT https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.DICT
 download_model model/English.RDR https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.RDR