(blacklist) Blacklist management

code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql

@@ -2,6 +2,7 @@
 CREATE TABLE IF NOT EXISTS EC_DOMAIN_BLACKLIST (
     ID INT PRIMARY KEY AUTO_INCREMENT,
     URL_DOMAIN VARCHAR(255) UNIQUE NOT NULL
+    COMMENT VARCHAR(255) DEFAULT NULL
 )
 CHARACTER SET utf8mb4
 COLLATE utf8mb4_unicode_ci;

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -24,6 +24,7 @@ import java.sql.SQLException;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 public class ControlService extends Service {
@@ -36,6 +37,7 @@ public class ControlService extends Service {
     private final EventLogService eventLogService;
     private final ApiKeyService apiKeyService;
     private final DomainComplaintService domainComplaintService;
+    private final ControlBlacklistService blacklistService;
     private final ControlActorService controlActorService;
     private final StaticResources staticResources;
     private final MessageQueueService messageQueueService;
@@ -54,6 +56,7 @@ public class ControlService extends Service {
                           ControlFileStorageService controlFileStorageService,
                           ApiKeyService apiKeyService,
                           DomainComplaintService domainComplaintService,
+                          ControlBlacklistService blacklistService,
                           ControlActionsService controlActionsService
                       ) throws IOException {
 
@@ -63,6 +66,7 @@ public class ControlService extends Service {
         this.eventLogService = eventLogService;
         this.apiKeyService = apiKeyService;
         this.domainComplaintService = domainComplaintService;
+        this.blacklistService = blacklistService;
 
         var indexRenderer = rendererFactory.renderer("control/index");
         var servicesRenderer = rendererFactory.renderer("control/services");
@@ -85,6 +89,7 @@ public class ControlService extends Service {
         var viewMessageRenderer = rendererFactory.renderer("control/view-message");
 
         var actionsViewRenderer = rendererFactory.renderer("control/actions");
+        var blacklistRenderer = rendererFactory.renderer("control/blacklist");
 
         this.controlActorService = controlActorService;
 
@@ -109,6 +114,7 @@ public class ControlService extends Service {
         final HtmlRedirect redirectToActors = new HtmlRedirect("/actors");
         final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys");
         final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage");
+        final HtmlRedirect redirectToBlacklist = new HtmlRedirect("/blacklist");
         final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints");
         final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue");
 
@@ -145,6 +151,11 @@ public class ControlService extends Service {
         Spark.post("/public/storage/specs", controlActorService::createCrawlSpecification, redirectToStorage);
         Spark.post("/public/storage/:fid/delete", controlFileStorageService::flagFileForDeletionRequest, redirectToStorage);
 
+        // Blacklist
+
+        Spark.get("/public/blacklist", this::blacklistModel, blacklistRenderer::render);
+        Spark.post("/public/blacklist", this::updateBlacklist, redirectToBlacklist);
+
         // API Keys
 
         Spark.get("/public/api-keys", this::apiKeysModel, apiKeysRenderer::render);
@@ -171,6 +182,22 @@ public class ControlService extends Service {
         monitors.subscribe(this::logMonitorStateChange);
     }
 
+    private Object blacklistModel(Request request, Response response) {
+        return Map.of("blacklist", blacklistService.lastNAdditions(100));
+    }
+
+    private Object updateBlacklist(Request request, Response response) {
+        var domain = new EdgeDomain(request.queryParams("domain"));
+        if ("add".equals(request.queryParams("act"))) {
+            var comment = Objects.requireNonNullElse(request.queryParams("comment"), "");
+            blacklistService.addToBlacklist(domain, comment);
+        } else if ("del".equals(request.queryParams("act"))) {
+            blacklistService.removeFromBlacklist(domain);
+        }
+
+        return "";
+    }
+
     private Object overviewModel(Request request, Response response) {
 
         return Map.of("processes", heartbeatService.getProcessHeartbeats(),

code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java

@@ -0,0 +1,6 @@
+package nu.marginalia.control.model;
+
+import nu.marginalia.model.EdgeDomain;
+
+public record BlacklistedDomainModel(EdgeDomain domain, String comment) {
+}

code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java

@@ -0,0 +1,79 @@
+package nu.marginalia.control.svc;
+
+import com.google.inject.Inject;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.control.model.BlacklistedDomainModel;
+import nu.marginalia.model.EdgeDomain;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ControlBlacklistService {
+
+    private final HikariDataSource dataSource;
+
+    @Inject
+    public ControlBlacklistService(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    public void addToBlacklist(EdgeDomain domain, String comment) {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN, COMMENT) VALUES (?, ?)
+                     """)) {
+            stmt.setString(1, domain.toString());
+            stmt.setString(2, comment);
+            stmt.executeUpdate();
+        }
+        catch (SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void removeFromBlacklist(EdgeDomain domain) {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?
+                     """)) {
+            stmt.setString(1, domain.toString());
+            stmt.addBatch();
+            stmt.setString(1, domain.domain);
+            stmt.addBatch();
+            stmt.executeBatch();
+        }
+        catch (SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public List<BlacklistedDomainModel> lastNAdditions(int n) {
+        final List<BlacklistedDomainModel> ret = new ArrayList<>(n);
+
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT URL_DOMAIN, COMMENT
+                     FROM EC_DOMAIN_BLACKLIST
+                     ORDER BY ID DESC
+                     LIMIT ?
+                     """)) {
+            stmt.setInt(1, n);
+
+            var rs = stmt.executeQuery();
+            while (rs.next()) {
+                ret.add(new BlacklistedDomainModel(
+                            new EdgeDomain(rs.getString("URL_DOMAIN")),
+                            rs.getString("COMMENT")
+                        )
+                );
+            }
+        }
+        catch (SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return ret;
+
+    }
+}

code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java

@@ -18,10 +18,14 @@ import java.util.Optional;
  */
 public class DomainComplaintService {
     private final HikariDataSource dataSource;
+    private final ControlBlacklistService blacklistService;
 
     @Inject
-    public DomainComplaintService(HikariDataSource dataSource) {
+    public DomainComplaintService(HikariDataSource dataSource,
+                                  ControlBlacklistService blacklistService
+    ) {
         this.dataSource = dataSource;
+        this.blacklistService = blacklistService;
     }
 
     public List<DomainComplaintModel> getComplaints() {
@@ -53,12 +57,13 @@ public class DomainComplaintService {
     }
 
     public void approveAppealBlacklisting(EdgeDomain domain) {
-        removeFromBlacklist(domain);
+        blacklistService.removeFromBlacklist(domain);
         setDecision(domain, "APPROVED");
     }
 
     public void blacklistDomain(EdgeDomain domain) {
-        addToBlacklist(domain);
+        blacklistService.addToBlacklist(domain, "Domain complaint");
+
         setDecision(domain, "BLACKLISTED");
     }
 
@@ -66,33 +71,7 @@ public class DomainComplaintService {
         setDecision(domain, "REJECTED");
     }
 
-    private void addToBlacklist(EdgeDomain domain) {
-        try (var conn = dataSource.getConnection();
-             var stmt = conn.prepareStatement("""
-                     INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN) VALUES (?)
-                     """)) {
-            stmt.setString(1, domain.toString());
-            stmt.executeUpdate();
-        }
-        catch (SQLException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
 
-    private void removeFromBlacklist(EdgeDomain domain) {
-        try (var conn = dataSource.getConnection();
-             var stmt = conn.prepareStatement("""
-                     DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?
-                     """)) {
-            stmt.setString(1, domain.toString());
-            stmt.addBatch();
-            stmt.setString(1, domain.domain);
-            stmt.executeBatch();
-        }
-        catch (SQLException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
 
     private void setDecision(EdgeDomain domain, String decision) {
         try (var conn = dataSource.getConnection();

code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb

@@ -0,0 +1,68 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Control Service</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="stylesheet" href="/style.css" />
+</head>
+<body>
+{{> control/partials/nav}}
+<section>
+    <h1>Blacklist</h1>
+
+    <p>
+        The blacklist is a list of sanctioned domains that will not be
+        crawled, indexed, or returned from the search results.
+    </p>
+
+    <table style="max-width: 80ch">
+        <tr>
+            <th>Description</th><th>Action</th>
+        </tr>
+        <tr>
+            <td><b>Add To Blacklist</b><p>
+                This will add the given domain to the blacklist.
+            </td>
+            <td>
+                <form method="post" action="/blacklist?act=add" onsubmit="return confirm('Confirm addition')">
+                    <label for="add-domain">Domain: </label>&nbsp;<input type="text" id="add-domain" name="domain" placeholder="Domain" /><br>
+                    <label for="comment">Comment: </label>&nbsp;<input type="text" id="comment" name="comment" value="Manually Added" />
+                    <br>
+                    <br>
+                    <input type="submit" value="Add">
+                </form>
+            </td>
+        </tr>
+        <tr>
+            <td><b>Remove from blacklist</b><p>
+                Remove the specified domain from the blacklist.  This will ensure that
+                the domain is not blacklisted, in doing so it may remove the root domain
+                from the blacklist as well.
+            </td>
+            <td>
+                <form method="post" action="/blacklist?act=del" onsubmit="return confirm('Confirm removal')">
+                    <label for="rm-domain">Domain: </label>&nbsp;<input type="text" id="rm-domain" name="domain" placeholder="Domain" />
+                    <br>
+                    <br>
+                    <input type="submit" value="Add">
+                </form>
+            </td>
+        </tr>
+    </table>
+
+    <h1>Recent Additions</h1>
+    <table>
+        <tr>
+            <th>Domain</th>
+            <th>Comment</th>
+        </tr>
+        {{#each blacklist}}
+        <tr>
+            <td>{{domain}}</td>
+            <td>{{comment}}</td>
+        </tr>
+        {{/each}}
+    </table>
+</section>
+</body>
+</html>