diff --git a/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdCollection.java b/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdCollection.java index a8403cc6..47cb4c62 100644 --- a/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdCollection.java +++ b/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdCollection.java @@ -1,9 +1,10 @@ package nu.marginalia.model.id; import java.util.Arrays; +import java.util.Iterator; import java.util.stream.IntStream; -public interface EdgeIdCollection { +public interface EdgeIdCollection extends Iterable> { int size(); boolean isEmpty(); int[] values(); @@ -12,6 +13,9 @@ public interface EdgeIdCollection { return Arrays.stream(values()); } + default Iterator> iterator() { + return Arrays.stream(values()).mapToObj(EdgeId::new).iterator(); + } default EdgeIdArray asArray() { return new EdgeIdArray<>(values()); } diff --git a/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdList.java b/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdList.java index 295854ec..98f651d1 100644 --- a/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdList.java +++ b/code/common/model/src/main/java/nu/marginalia/model/id/EdgeIdList.java @@ -5,7 +5,9 @@ import gnu.trove.list.array.TIntArrayList; import java.util.stream.IntStream; -public record EdgeIdList (TIntArrayList list) implements EdgeIdCollection, EdgeIdCollectionMutable { +public record EdgeIdList (TIntArrayList list) implements + EdgeIdCollection, + EdgeIdCollectionMutable { public EdgeIdList(int... values) { this(new TIntArrayList(values)); } public static EdgeIdList gather(IntStream stream) { diff --git a/code/services-core/control-service/build.gradle b/code/services-core/control-service/build.gradle index af67d328..b081ba65 100644 --- a/code/services-core/control-service/build.gradle +++ b/code/services-core/control-service/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation project(':code:api:search-api') implementation project(':code:api:index-api') implementation project(':code:api:process-mqapi') - + implementation project(':code:features-search:screenshots') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index 6e64d68b..0fc1a178 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -2,6 +2,7 @@ package nu.marginalia.control; import com.google.gson.Gson; import com.google.inject.Inject; +import gnu.trove.list.array.TIntArrayList; import nu.marginalia.client.ServiceMonitors; import nu.marginalia.control.actor.Actor; import nu.marginalia.control.model.*; @@ -10,7 +11,9 @@ import nu.marginalia.db.storage.model.FileStorageId; import nu.marginalia.db.storage.model.FileStorageType; import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.gson.GsonFactory; +import nu.marginalia.model.id.EdgeIdList; import nu.marginalia.renderer.RendererFactory; +import nu.marginalia.screenshot.ScreenshotService; import nu.marginalia.service.server.*; import org.eclipse.jetty.util.StringUtil; import org.slf4j.Logger; @@ -35,6 +38,7 @@ public class ControlService extends Service { private final ApiKeyService apiKeyService; private final DomainComplaintService domainComplaintService; private final ControlBlacklistService blacklistService; + private final RandomExplorationService randomExplorationService; private final ControlActorService controlActorService; private final StaticResources staticResources; private final MessageQueueService messageQueueService; @@ -54,7 +58,9 @@ public class ControlService extends Service { ApiKeyService apiKeyService, DomainComplaintService domainComplaintService, ControlBlacklistService blacklistService, - ControlActionsService controlActionsService + ControlActionsService controlActionsService, + ScreenshotService screenshotService, + RandomExplorationService randomExplorationService ) throws IOException { super(params); @@ -64,6 +70,7 @@ public class ControlService extends Service { this.apiKeyService = apiKeyService; this.domainComplaintService = domainComplaintService; this.blacklistService = blacklistService; + this.randomExplorationService = randomExplorationService; var indexRenderer = rendererFactory.renderer("control/index"); var eventsRenderer = rendererFactory.renderer("control/events"); @@ -75,6 +82,7 @@ public class ControlService extends Service { var storageSpecsRenderer = rendererFactory.renderer("control/storage-specs"); var storageCrawlsRenderer = rendererFactory.renderer("control/storage-crawls"); var storageProcessedRenderer = rendererFactory.renderer("control/storage-processed"); + var reviewRandomDomainsRenderer = rendererFactory.renderer("control/review-random-domains"); var apiKeysRenderer = rendererFactory.renderer("control/api-keys"); var domainComplaintsRenderer = rendererFactory.renderer("control/domain-complaints"); @@ -117,6 +125,9 @@ public class ControlService extends Service { final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints"); final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue"); + // Needed to be able to show website screenshots + Spark.get("/public/screenshot/:id", screenshotService::serveScreenshotRequest); + // FSMs Spark.post("/public/fsms/:fsm/start", controlActorService::startFsm, redirectToActors); @@ -178,11 +189,51 @@ public class ControlService extends Service { Spark.post("/public/actions/flush-api-caches", controlActionsService::flushApiCaches, redirectToActors); Spark.post("/public/actions/truncate-links-database", controlActionsService::truncateLinkDatabase, redirectToActors); + // Review Random Domains + Spark.get("/public/review-random-domains", this::reviewRandomDomainsModel, reviewRandomDomainsRenderer::render); + + Spark.post("/public/review-random-domains", this::reviewRandomDomainsAction); + + Spark.get("/public/:resource", this::serveStatic); monitors.subscribe(this::logMonitorStateChange); } + private Object reviewRandomDomainsModel(Request request, Response response) throws SQLException { + String afterVal = Objects.requireNonNullElse(request.queryParams("after"), "0"); + int after = Integer.parseInt(afterVal); + var domains = randomExplorationService.getDomains(after, 25); + int nextAfter = domains.stream().mapToInt(RandomExplorationService.RandomDomainResult::id).max().orElse(Integer.MAX_VALUE); + + return Map.of("domains", domains, + "after", nextAfter); + + } + + private Object reviewRandomDomainsAction(Request request, Response response) throws SQLException { + TIntArrayList idList = new TIntArrayList(); + + request.queryParams().forEach(key -> { + if (key.startsWith("domain-")) { + String value = request.queryParams(key); + if ("on".equalsIgnoreCase(value)) { + int id = Integer.parseInt(key.substring(7)); + idList.add(id); + } + } + }); + + randomExplorationService.removeRandomDomains(new EdgeIdList<>(idList.toArray())); + + String after = request.queryParams("after"); + + return """ + + + """.formatted(after); + } + private Object blacklistModel(Request request, Response response) { return Map.of("blacklist", blacklistService.lastNAdditions(100)); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java new file mode 100644 index 00000000..c861e961 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/RandomExplorationService.java @@ -0,0 +1,62 @@ +package nu.marginalia.control.svc; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.model.EdgeDomain; +import nu.marginalia.model.id.EdgeIdList; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class RandomExplorationService { + + private final HikariDataSource dataSource; + + @Inject + public RandomExplorationService(HikariDataSource dataSource) { + this.dataSource = dataSource; + } + + public void removeRandomDomains(EdgeIdList ids) throws SQLException { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + DELETE FROM EC_RANDOM_DOMAINS + WHERE DOMAIN_ID = ? + AND DOMAIN_SET = 0 + """)) + { + for (var id : ids) { + stmt.setInt(1, id.id()); + stmt.addBatch(); + } + stmt.executeBatch(); + if (!conn.getAutoCommit()) { + conn.commit(); + } + } + } + + public List getDomains(int afterId, int numResults) throws SQLException { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT DOMAIN_ID, DOMAIN_NAME FROM EC_RANDOM_DOMAINS + INNER JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_ID + WHERE DOMAIN_ID >= ? + LIMIT ? + """)) + { + List ret = new ArrayList<>(numResults); + stmt.setInt(1, afterId); + stmt.setInt(2, numResults); + var rs = stmt.executeQuery(); + while (rs.next()) { + ret.add(new RandomDomainResult(rs.getInt(1), rs.getString(2))); + } + return ret; + } + } + + + public record RandomDomainResult(int id, String domainName) {} +} diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb b/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb index 39d3541d..bd34803b 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/partials/nav.hdb @@ -5,6 +5,7 @@
  • API Keys
  • Blacklist
  • Complaints
    • +
  • Random Exploration
  • System
  • Actions
  • Storage
    • diff --git a/code/services-core/control-service/src/main/resources/templates/control/review-random-domains.hdb b/code/services-core/control-service/src/main/resources/templates/control/review-random-domains.hdb new file mode 100644 index 00000000..7802232f --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/review-random-domains.hdb @@ -0,0 +1,36 @@ + + + + Control Service + + + + +{{> control/partials/nav}} +
    +

    Domain Review

    +
    + + + + + {{#each domains}} + + + + + + {{/each}} + + + +
    ActionDomain NameScreenshot
    + + + {{domainName}} + + +
    + +
    +
    \ No newline at end of file