(control-service) Basic GUI for deleting bad links from exploration mode

code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdCollection.java

@@ -1,9 +1,10 @@
 package nu.marginalia.model.id;
 
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.stream.IntStream;
 
-public interface EdgeIdCollection<T> {
+public interface EdgeIdCollection<T> extends Iterable<EdgeId<T>> {
     int size();
     boolean isEmpty();
     int[] values();
@@ -12,6 +13,9 @@ public interface EdgeIdCollection<T> {
         return Arrays.stream(values());
     }
 
+    default Iterator<EdgeId<T>> iterator() {
+        return Arrays.stream(values()).mapToObj(EdgeId<T>::new).iterator();
+    }
     default EdgeIdArray<T> asArray() {
         return new EdgeIdArray<>(values());
     }

code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdList.java

@@ -5,7 +5,9 @@ import gnu.trove.list.array.TIntArrayList;
 
 import java.util.stream.IntStream;
 
-public record EdgeIdList<T> (TIntArrayList list) implements EdgeIdCollection<T>, EdgeIdCollectionMutable<T> {
+public record EdgeIdList<T> (TIntArrayList list) implements
+        EdgeIdCollection<T>,
+        EdgeIdCollectionMutable<T> {
 
     public EdgeIdList(int... values) { this(new TIntArrayList(values)); }
     public static <T> EdgeIdList<T> gather(IntStream stream) {

code/services-core/control-service/build.gradle

@@ -35,7 +35,7 @@ dependencies {
     implementation project(':code:api:search-api')
     implementation project(':code:api:index-api')
     implementation project(':code:api:process-mqapi')
-
+    implementation project(':code:features-search:screenshots')
 
     implementation libs.lombok
     annotationProcessor libs.lombok

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -2,6 +2,7 @@ package nu.marginalia.control;
 
 import com.google.gson.Gson;
 import com.google.inject.Inject;
+import gnu.trove.list.array.TIntArrayList;
 import nu.marginalia.client.ServiceMonitors;
 import nu.marginalia.control.actor.Actor;
 import nu.marginalia.control.model.*;
@@ -10,7 +11,9 @@ import nu.marginalia.db.storage.model.FileStorageId;
 import nu.marginalia.db.storage.model.FileStorageType;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.model.gson.GsonFactory;
+import nu.marginalia.model.id.EdgeIdList;
 import nu.marginalia.renderer.RendererFactory;
+import nu.marginalia.screenshot.ScreenshotService;
 import nu.marginalia.service.server.*;
 import org.eclipse.jetty.util.StringUtil;
 import org.slf4j.Logger;
@@ -35,6 +38,7 @@ public class ControlService extends Service {
     private final ApiKeyService apiKeyService;
     private final DomainComplaintService domainComplaintService;
     private final ControlBlacklistService blacklistService;
+    private final RandomExplorationService randomExplorationService;
     private final ControlActorService controlActorService;
     private final StaticResources staticResources;
     private final MessageQueueService messageQueueService;
@@ -54,7 +58,9 @@ public class ControlService extends Service {
                           ApiKeyService apiKeyService,
                           DomainComplaintService domainComplaintService,
                           ControlBlacklistService blacklistService,
-                          ControlActionsService controlActionsService
+                          ControlActionsService controlActionsService,
+                          ScreenshotService screenshotService,
+                          RandomExplorationService randomExplorationService
                       ) throws IOException {
 
         super(params);
@@ -64,6 +70,7 @@ public class ControlService extends Service {
         this.apiKeyService = apiKeyService;
         this.domainComplaintService = domainComplaintService;
         this.blacklistService = blacklistService;
+        this.randomExplorationService = randomExplorationService;
 
         var indexRenderer = rendererFactory.renderer("control/index");
         var eventsRenderer = rendererFactory.renderer("control/events");
@@ -75,6 +82,7 @@ public class ControlService extends Service {
         var storageSpecsRenderer = rendererFactory.renderer("control/storage-specs");
         var storageCrawlsRenderer = rendererFactory.renderer("control/storage-crawls");
         var storageProcessedRenderer = rendererFactory.renderer("control/storage-processed");
+        var reviewRandomDomainsRenderer = rendererFactory.renderer("control/review-random-domains");
 
         var apiKeysRenderer = rendererFactory.renderer("control/api-keys");
         var domainComplaintsRenderer = rendererFactory.renderer("control/domain-complaints");
@@ -117,6 +125,9 @@ public class ControlService extends Service {
         final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints");
         final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue");
 
+        // Needed to be able to show website screenshots
+        Spark.get("/public/screenshot/:id", screenshotService::serveScreenshotRequest);
+
         // FSMs
 
         Spark.post("/public/fsms/:fsm/start", controlActorService::startFsm, redirectToActors);
@@ -178,11 +189,51 @@ public class ControlService extends Service {
         Spark.post("/public/actions/flush-api-caches", controlActionsService::flushApiCaches, redirectToActors);
         Spark.post("/public/actions/truncate-links-database", controlActionsService::truncateLinkDatabase, redirectToActors);
 
+        // Review Random Domains
+        Spark.get("/public/review-random-domains", this::reviewRandomDomainsModel, reviewRandomDomainsRenderer::render);
+
+        Spark.post("/public/review-random-domains", this::reviewRandomDomainsAction);
+
+
         Spark.get("/public/:resource", this::serveStatic);
 
         monitors.subscribe(this::logMonitorStateChange);
     }
 
+    private Object reviewRandomDomainsModel(Request request, Response response) throws SQLException {
+        String afterVal = Objects.requireNonNullElse(request.queryParams("after"), "0");
+        int after = Integer.parseInt(afterVal);
+        var domains = randomExplorationService.getDomains(after, 25);
+        int nextAfter = domains.stream().mapToInt(RandomExplorationService.RandomDomainResult::id).max().orElse(Integer.MAX_VALUE);
+
+        return Map.of("domains", domains,
+                        "after", nextAfter);
+
+    }
+
+    private Object reviewRandomDomainsAction(Request request, Response response) throws SQLException {
+        TIntArrayList idList = new TIntArrayList();
+
+        request.queryParams().forEach(key -> {
+            if (key.startsWith("domain-")) {
+                String value = request.queryParams(key);
+                if ("on".equalsIgnoreCase(value)) {
+                    int id = Integer.parseInt(key.substring(7));
+                    idList.add(id);
+                }
+            }
+        });
+
+        randomExplorationService.removeRandomDomains(new EdgeIdList<>(idList.toArray()));
+
+        String after = request.queryParams("after");
+
+        return """
+                <?doctype html>
+                <html><head><meta http-equiv="refresh" content="0;URL='/review-random-domains?after=%s'" /></head></html>
+                """.formatted(after);
+    }
+
 
     private Object blacklistModel(Request request, Response response) {
         return Map.of("blacklist", blacklistService.lastNAdditions(100));

code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java

@@ -0,0 +1,62 @@
+package nu.marginalia.control.svc;
+
+import com.google.inject.Inject;
+import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.model.EdgeDomain;
+import nu.marginalia.model.id.EdgeIdList;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class RandomExplorationService {
+
+    private final HikariDataSource dataSource;
+
+    @Inject
+    public RandomExplorationService(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    public void removeRandomDomains(EdgeIdList<EdgeDomain> ids) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     DELETE FROM EC_RANDOM_DOMAINS
+                     WHERE DOMAIN_ID = ?
+                     AND DOMAIN_SET = 0
+                     """))
+        {
+            for (var id : ids) {
+                stmt.setInt(1, id.id());
+                stmt.addBatch();
+            }
+            stmt.executeBatch();
+            if (!conn.getAutoCommit()) {
+                conn.commit();
+            }
+        }
+    }
+
+    public List<RandomDomainResult> getDomains(int afterId, int numResults) throws SQLException {
+        try (var conn = dataSource.getConnection();
+             var stmt = conn.prepareStatement("""
+                     SELECT DOMAIN_ID, DOMAIN_NAME FROM EC_RANDOM_DOMAINS
+                     INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID
+                     WHERE DOMAIN_ID >= ?
+                     LIMIT ?
+                     """))
+        {
+            List<RandomDomainResult> ret = new ArrayList<>(numResults);
+            stmt.setInt(1, afterId);
+            stmt.setInt(2, numResults);
+            var rs = stmt.executeQuery();
+            while (rs.next()) {
+                ret.add(new RandomDomainResult(rs.getInt(1), rs.getString(2)));
+            }
+            return ret;
+        }
+    }
+
+
+    public record RandomDomainResult(int id, String domainName) {}
+}

code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb

@@ -5,6 +5,7 @@
         <li><a href="/api-keys" title="Create or remove API keys">API Keys</a></li>
         <li><a href="/blacklist" title="Add or remove website sanctions">Blacklist</a></li>
         <li><a href="/complaints" title="View and act on user complaints">Complaints</a></li>
+        <li><a href="/review-random-domains" title="Review random domains list">Random Exploration</a></li>
         <li>System</li>
         <li><a href="/actions" title="Trigger system actions">Actions</a></li>
         <li><a href="/storage" title="View file storage, trigger crawling and processing">Storage</a></li>

code/services-core/control-service/src/main/resources/templates/control/review-random-domains.hdb

@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Control Service</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="stylesheet" href="/style.css" />
+</head>
+<body>
+{{> control/partials/nav}}
+<section>
+    <h1>Domain Review</h1>
+    <form method="POST" action="/review-random-domains">
+    <table>
+        <tr>
+            <th>Action</th><th>Domain Name</th><th>Screenshot</th>
+        </tr>
+        {{#each domains}}
+        <tr>
+            <td>
+                <input type="checkbox" name="domain-{{id}}" id="domain-{{id}}"/> <label for="domain-{{id}}">Remove</label>
+            </td>
+            <td>
+                <a href="https://{{domainName}}">{{domainName}}</a>
+            </td>
+            <td>
+                <img src="/screenshot/{{id}}" style="max-width: 100%; max-height: 100px"/>
+            </td>
+        </tr>
+        {{/each}}
+        <tr>
+            <td colspan="3"><input type="Submit" value="Process"></td>
+        </tr>
+    </table>
+        <input type="hidden" name="after" value="{{after}}" />
+    </form>
+</section>