(control) Add basic api key management

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -12,6 +12,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.persistence.MqPersistence;
 import nu.marginalia.renderer.RendererFactory;
 import nu.marginalia.service.server.*;
+import org.eclipse.jetty.util.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import spark.Request;
@@ -30,6 +31,7 @@ public class ControlService extends Service {
     private final ServiceMonitors monitors;
     private final HeartbeatService heartbeatService;
     private final EventLogService eventLogService;
+    private final ApiKeyService apiKeyService;
     private final ControlActorService controlActorService;
     private final StaticResources staticResources;
     private final MessageQueueViewService messageQueueViewService;
@@ -46,6 +48,7 @@ public class ControlService extends Service {
                           StaticResources staticResources,
                           MessageQueueViewService messageQueueViewService,
                           ControlFileStorageService controlFileStorageService,
+                          ApiKeyService apiKeyService,
                           MqPersistence persistence
                       ) throws IOException {
 
@@ -53,6 +56,7 @@ public class ControlService extends Service {
         this.monitors = monitors;
         this.heartbeatService = heartbeatService;
         this.eventLogService = eventLogService;
+        this.apiKeyService = apiKeyService;
 
         var indexRenderer = rendererFactory.renderer("control/index");
         var servicesRenderer = rendererFactory.renderer("control/services");
@@ -64,6 +68,8 @@ public class ControlService extends Service {
         var storageCrawlsRenderer = rendererFactory.renderer("control/storage-crawls");
         var storageProcessedRenderer = rendererFactory.renderer("control/storage-processed");
 
+        var apiKeysRenderer = rendererFactory.renderer("control/api-keys");
+
         var storageDetailsRenderer = rendererFactory.renderer("control/storage-details");
         var updateMessageStateRenderer = rendererFactory.renderer("control/dialog-update-message-state");
 
@@ -95,6 +101,7 @@ public class ControlService extends Service {
 
         final HtmlRedirect redirectToServices = new HtmlRedirect("/services");
         final HtmlRedirect redirectToProcesses = new HtmlRedirect("/actors");
+        final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys");
         final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage");
 
         Spark.post("/public/fsms/:fsm/start", controlActorService::startFsm, redirectToProcesses);
@@ -107,6 +114,12 @@ public class ControlService extends Service {
         Spark.post("/public/storage/specs", controlActorService::createCrawlSpecification, redirectToStorage);
         Spark.post("/public/storage/:fid/delete", controlFileStorageService::flagFileForDeletionRequest, redirectToStorage);
 
+        Spark.get("/public/api-keys", this::apiKeysModel, apiKeysRenderer::render);
+        Spark.post("/public/api-keys", this::createApiKey, redirectToApiKeys);
+        Spark.delete("/public/api-keys/:key", this::deleteApiKey, redirectToApiKeys);
+        // HTML forms don't support the DELETE verb :-(
+        Spark.post("/public/api-keys/:key/delete", this::deleteApiKey, redirectToApiKeys);
+
         Spark.get("/public/message/:id/state", (rq, rsp) -> persistence.getMessage(Long.parseLong(rq.params("id"))), updateMessageStateRenderer::render);
         Spark.post("/public/message/:id/state", (rq, rsp) -> {
             MqMessageState state = MqMessageState.valueOf(rq.queryParams("state"));
@@ -120,6 +133,37 @@ public class ControlService extends Service {
         monitors.subscribe(this::logMonitorStateChange);
     }
 
+    private Object createApiKey(Request request, Response response) {
+        String license = request.queryParams("license");
+        String name = request.queryParams("name");
+        String email = request.queryParams("email");
+        int rate = Integer.parseInt(request.queryParams("rate"));
+
+        if (StringUtil.isBlank(license) ||
+            StringUtil.isBlank(name) ||
+            StringUtil.isBlank(email) ||
+            rate <= 0)
+        {
+            response.status(400);
+            return "";
+        }
+
+        apiKeyService.addApiKey(license, name, email, rate);
+
+        return "";
+    }
+
+    private Object deleteApiKey(Request request, Response response) {
+        String licenseKey = request.params("key");
+        apiKeyService.deleteApiKey(licenseKey);
+        return "";
+    }
+
+    private Object apiKeysModel(Request request, Response response) {
+        return Map.of("apikeys", apiKeyService.getApiKeys());
+    }
+
+
     @Override
     public void logRequest(Request request) {
         if ("GET".equals(request.requestMethod()))

code/services-core/control-service/src/main/java/nu/marginalia/control/model/ApiKeyModel.java

@@ -0,0 +1,2 @@
+package nu.marginalia.control.model;
+public record ApiKeyModel(String licenseKey, String license, String name, String email, int rate) {}

code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ApiKeyService.java

@@ -0,0 +1,93 @@
+package nu.marginalia.control.svc;
+
+import com.google.inject.Inject;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.model.ApiKeyModel;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+public class ApiKeyService {
+
+    private final HikariDataSource dataSource;
+
+    @Inject
+    public ApiKeyService(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    public List<ApiKeyModel> getApiKeys() {
+        try (var conn = dataSource.getConnection()) {
+            try (var stmt = conn.prepareStatement("""
+                    SELECT LICENSE_KEY, LICENSE, NAME, EMAIL, RATE FROM EC_API_KEY
+                    """)) {
+                List<ApiKeyModel> ret = new ArrayList<>(100);
+                var rs = stmt.executeQuery();
+                while (rs.next()) {
+                    ret.add(new ApiKeyModel(
+                            rs.getString("LICENSE_KEY"),
+                            rs.getString("LICENSE"),
+                            rs.getString("NAME"),
+                            rs.getString("EMAIL"),
+                            rs.getInt("RATE")));
+                }
+                return ret;
+            }
+        }
+        catch (SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public ApiKeyModel addApiKey(String license, String name, String email, int rate) {
+        try (var conn = dataSource.getConnection()) {
+            try (var insertStmt = conn.prepareStatement("""
+                    INSERT INTO EC_API_KEY (LICENSE_KEY, LICENSE, NAME, EMAIL, RATE) SELECT SHA(?), ?, ?, ?, ?
+                    """);
+                 // we could do SELECT SHA(?) here I guess if performance was a factor, but it's not
+                 var queryStmt = conn.prepareStatement("SELECT LICENSE_KEY FROM EC_API_KEY WHERE LICENSE_KEY = SHA(?)")
+            ) {
+                final String seedString = UUID.randomUUID() + "-" + name + "-" + email;
+
+                insertStmt.setString(1, seedString);
+                insertStmt.setString(2, license);
+                insertStmt.setString(3, name);
+                insertStmt.setString(4, email);
+                insertStmt.setInt(5, rate);
+                insertStmt.executeUpdate();
+
+                queryStmt.setString(1, seedString);
+                var rs = queryStmt.executeQuery();
+                if (rs.next()) {
+                    return new ApiKeyModel(
+                            rs.getString("LICENSE_KEY"),
+                            license,
+                            name,
+                            email,
+                            rate);
+                }
+
+                throw new RuntimeException("Failed to insert key");
+            }
+        }
+        catch (SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void deleteApiKey(String key) {
+        try (var conn = dataSource.getConnection()) {
+            try (var stmt = conn.prepareStatement("""
+                    DELETE FROM EC_API_KEY WHERE LICENSE_KEY = ?
+                    """)) {
+                stmt.setString(1, key);
+                stmt.executeUpdate();
+            }
+        }
+        catch (SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+}

code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb

@@ -0,0 +1,61 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Control Service</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="stylesheet" href="/style.css" />
+</head>
+<body>
+{{> control/partials/nav}}
+<section>
+
+    <h1>API Keys</h1>
+    <table id="apikeys">
+        <tr>
+            <th colspan="3">Key</th>
+            <th>&nbsp;</th>
+        </tr>
+        <tr>
+            <th>License</th>
+            <th>Name</th>
+            <th>Contact</th>
+            <th>Rate</th>
+        </tr>
+    {{#each apikeys}}
+        <tr>
+            <td colspan="3">{{licenseKey}}</td>
+            <td>
+                <form method="post" action="/api-keys/{{licenseKey}}/delete" onsubmit="return confirm('Confirm deletion of {{licenseKey}}')">
+                    <input type="submit" value="Delete" />
+                </form>
+            </td>
+        </tr>
+        <tr>
+            <td>{{license}}</td>
+            <td>{{name}}</td>
+            <td>{{email}}</td>
+            <td>{{rate}}</td>
+        </tr>
+    {{/each}}
+    </table>
+    <h2>Add New</h2>
+    <form action="/api-keys" method="post">
+        <label for="name">Name</label><br>
+        <input type="text" name="name" id="name" /><br>
+        <label for="email">Contact Email</label><br>
+        <input type="text" name="email" id="email" /><br>
+        <label for="license">License</label><br>
+        <input type="text" name="license" id="license" value="CC-BY-NC-SA 4.0" /><br>
+        <label for="rate">Rate</label><br>
+        <input type="text" name="rate" id="rate" value="15" /><br><br>
+        <input type="submit" value="Create" />
+    </form>
+</section>
+</body>
+<script src="/refresh.js"></script>
+<script>
+    window.setInterval(() => {
+        refresh(["apikeys"]);
+    }, 2000);
+</script>
+</html>

code/services-core/control-service/src/main/resources/templates/control/dialog-update-message-state.hdb

@@ -12,16 +12,16 @@ erase information about its owner, and inboxes will consider the message new aga
 <form method="post" action="/message/{{msgId}}/state">
     <label for="msgId">msgId</label><br>
     <input type="text" disabled id="msgId" name="msgId" value="{{msgId}}">
-    <br><br>
+    <br>
     <label for="relatedId">relatedId</label><br>
     <input type="text" disabled id="relatedId" name="relatedId" value="{{relatedId}}">
-    <br><br>
+    <br>
     <label for="function">function</label><br>
     <input type="text" disabled id="function" name="function" value="{{function}}">
-    <br><br>
+    <br>
     <label for="payload">payload</label><br>
     <input type="text" disabled id="payload" name="payload" value="{{payload}}">
-    <br><br>
+    <br>
     <label for="oldState">current state</label><br>
     <input type="text" disabled id="oldState" name="oldState" value="{{state}}">
     <br>
@@ -37,5 +37,8 @@ erase information about its owner, and inboxes will consider the message new aga
 
     <input type="submit" value="Update">
 </form>
+    <p>Note that while setting a message to NEW or in some instances ACK typically causes an Actor
+    to act on the message, setting a message in ACK to ERR or DEAD will not stop action, but only
+    prevent resumption of action.  To stop a running actor, use the Actors view and press the toggle.</p>
 </section>
 </html>

code/services-core/control-service/src/main/resources/templates/control/partials/actors-table.hdb

@@ -15,7 +15,13 @@
                     action="/fsms/{{name}}/stop"
                     method="post"
                     onsubmit="return toggleActorSwitch('{{name}}')">
-                    <input type="submit" value="On" class="toggle-switch-on" id="toggle-{{name}}-button">
+                    <input
+                            type="submit"
+                            value="On"
+                            class="toggle-switch-on"
+                            id="toggle-{{name}}-button"
+                            title="Terminate the actor"
+                    >
                 </form>
             {{/unless}}
             {{#if terminal}}
@@ -32,6 +38,7 @@
                         {{/unless}}
                         {{#if canStart}}
                         value="Off"
+                        title="Start the actor"
                         {{/if}}
                         class="toggle-switch-off"
 

code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb

@@ -4,5 +4,8 @@
         <li><a href="/services">Services</a></li>
         <li><a href="/actors">Actors</a></li>
         <li><a href="/storage">Storage</a></li>
+        <li><a href="/api-keys">API Keys</a></li>
+        <li><a href="/blacklist">Blacklist</a></li>
+        <li><a href="/complaints">Complaints</a></li>
     </ul>
 </nav>

code/services-core/control-service/src/main/resources/templates/control/partials/storage-table.hdb

@@ -4,20 +4,18 @@
         <th>Type</th>
         <th>Name</th>
         <th>Path</th>
-        <th>Must Clean</th>
         <th>Permit Temp</th>
     </tr>
     <tr>
         <td>{{base.type}}</td>
         <td>{{base.name}}</td>
         <td>{{base.path}}</td>
-        <td>{{base.mustClean}}</td>
         <td>{{base.permitTemp}}</td>
     </tr>
     <tr>
         <th></th>
         <th>Type</th>
-        <th colspan="2">Path</th>
+        <th>Path</th>
         <th>Description</th>
     </tr>
     {{#each storage}}
@@ -26,7 +24,7 @@
         <a href="/storage/{{storage.id}}">Info</a>
         </td>
         <td>{{storage.type}}</td>
-        <td colspan="2">{{storage.path}}</td>
+        <td>{{storage.path}}</td>
         <td>{{storage.description}}</td>
     </tr>
     {{/each}}

code/services-core/control-service/src/main/resources/templates/control/storage-details.hdb

@@ -47,22 +47,22 @@
       <h2>Actions</h2>
       {{#with storage.self}}
       {{#if isCrawlable}}
-      <form method="post" action="/storage/{{storage.id}}/crawl">
+      <form method="post" action="/storage/{{storage.id}}/crawl" onsubmit="return confirm('Confirm crawling of {{storage.path}}')">
           Perform a full re-crawl of this data: <button type="submit">Crawl</button> <br>
       </form>
       {{/if}}
       {{#if isLoadable}}
-      <form method="post" action="/storage/{{storage.id}}/load">
+      <form method="post" action="/storage/{{storage.id}}/load" onsubmit="return confirm('Confirm loading of {{storage.path}}')">
           Load this data into index: <button type="submit">Load</button> <br>
       </form>
       {{/if}}
       {{#if isConvertible}}
-      <form method="post" action="/storage/{{storage.id}}/process">
+      <form method="post" action="/storage/{{storage.id}}/process" onsubmit="return confirm('Confirm processing of {{storage.path}}')">
           Process and load this data into index: <button type="submit">Process</button> <br>
       </form>
       {{/if}}
       {{#if isRecrawlable}}
-      <form method="post" action="/storage/{{storage.id}}/recrawl">
+      <form method="post" action="/storage/{{storage.id}}/recrawl" onsubmit="return confirm('Confirm re-crawling of {{storage.path}}')">
           Perform a re-crawl of this data: <button type="submit">Recrawl</button><br>
       </form>
       {{/if}}

code/services-core/control-service/src/main/resources/templates/control/storage-overview.hdb

@@ -16,20 +16,18 @@
         <th>Type</th>
         <th>Name</th>
         <th>Path</th>
-        <th>Must Clean</th>
         <th>Permit Temp</th>
       </tr>
       <tr>
         <td>{{base.type}}</td>
         <td>{{base.name}}</td>
         <td>{{base.path}}</td>
-        <td>{{base.mustClean}}</td>
         <td>{{base.permitTemp}}</td>
       </tr>
       <tr>
       <th></th>
       <th>Type</th>
-      <th colspan="2">Path</th>
+      <th>Path</th>
       <th>Description</th>
       </tr>
       {{#each storage}}
@@ -37,7 +35,7 @@
             <td>
             </td>
             <td>{{storage.type}}</td>
-            <td colspan="2">{{storage.path}}</td>
+            <td>{{storage.path}}</td>
             <td>{{storage.description}}</td>
         </tr>
       {{/each}}

code/services-core/control-service/src/test/java/nu/marginalia/control/svc/ApiKeyServiceTest.java

@@ -0,0 +1,94 @@
+package nu.marginalia.control.svc;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.model.ApiKeyModel;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Execution;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+@Testcontainers
+@Execution(SAME_THREAD)
+@Tag("slow")
+public class ApiKeyServiceTest {
+    @Container
+    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
+            .withDatabaseName("WMSA_prod")
+            .withUsername("wmsa")
+            .withPassword("wmsa")
+            .withInitScript("db/migration/V23_06_0_006__api_key.sql")
+            .withNetworkAliases("mariadb");
+
+    static HikariDataSource dataSource;
+    @BeforeAll
+    public static void setup() {
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(mariaDBContainer.getJdbcUrl());
+        config.setUsername("wmsa");
+        config.setPassword("wmsa");
+
+        dataSource = new HikariDataSource(config);
+    }
+
+    @AfterAll
+    public static void tearDown() {
+        dataSource.close();
+        mariaDBContainer.close();
+    }
+
+    @Test
+    void getKeys() {
+        var apiKeyService = new ApiKeyService(dataSource);
+        apiKeyService.addApiKey("public domain", "bob dobbs", "bob@dobbstown.com", 30);
+        apiKeyService.addApiKey("public domain", "connie dobbs", "cdobbs@dobbstown.com", 15);
+
+        var keys = apiKeyService.getApiKeys();
+        System.out.println(keys);
+        assertEquals(2, keys.size());
+    }
+
+    @Test
+    void addApiKey() {
+        var apiKeyService = new ApiKeyService(dataSource);
+        apiKeyService.addApiKey("public domain", "bob dobbs", "bob@dobbstown.com", 30);
+
+        var keys = apiKeyService.getApiKeys();
+
+        System.out.println(keys);
+        assertEquals(1, keys.size());
+
+        var key = keys.get(0);
+
+        assertEquals("public domain", key.license());
+        assertEquals("bob dobbs", key.name());
+        assertEquals("bob@dobbstown.com", key.email());
+        assertEquals(30, key.rate());
+        assertNotNull(key.licenseKey());
+    }
+
+    @Test
+    void deleteApiKey() {
+        var apiKeyService = new ApiKeyService(dataSource);
+        apiKeyService.addApiKey("public domain", "bob dobbs", "bob@dobbstown.com", 30);
+
+        List<ApiKeyModel> keys = apiKeyService.getApiKeys();
+
+        assertEquals(1, keys.size());
+
+        String licenseKey=  keys.get(0).licenseKey();
+        apiKeyService.deleteApiKey(licenseKey);
+
+        keys = apiKeyService.getApiKeys();
+        assertEquals(0, keys.size());
+    }
+}