diff --git a/code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql b/code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql index e46161bc..d05d8e9d 100644 --- a/code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql +++ b/code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS EC_DOMAIN_BLACKLIST ( ID INT PRIMARY KEY AUTO_INCREMENT, URL_DOMAIN VARCHAR(255) UNIQUE NOT NULL + COMMENT VARCHAR(255) DEFAULT NULL ) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index 671cd584..2b49b249 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -24,6 +24,7 @@ import java.sql.SQLException; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; public class ControlService extends Service { @@ -36,6 +37,7 @@ public class ControlService extends Service { private final EventLogService eventLogService; private final ApiKeyService apiKeyService; private final DomainComplaintService domainComplaintService; + private final ControlBlacklistService blacklistService; private final ControlActorService controlActorService; private final StaticResources staticResources; private final MessageQueueService messageQueueService; @@ -54,6 +56,7 @@ public class ControlService extends Service { ControlFileStorageService controlFileStorageService, ApiKeyService apiKeyService, DomainComplaintService domainComplaintService, + ControlBlacklistService blacklistService, ControlActionsService controlActionsService ) throws IOException { @@ -63,6 +66,7 @@ public class ControlService extends Service { this.eventLogService = eventLogService; this.apiKeyService = apiKeyService; this.domainComplaintService = domainComplaintService; + this.blacklistService = blacklistService; var indexRenderer = rendererFactory.renderer("control/index"); var servicesRenderer = rendererFactory.renderer("control/services"); @@ -85,6 +89,7 @@ public class ControlService extends Service { var viewMessageRenderer = rendererFactory.renderer("control/view-message"); var actionsViewRenderer = rendererFactory.renderer("control/actions"); + var blacklistRenderer = rendererFactory.renderer("control/blacklist"); this.controlActorService = controlActorService; @@ -109,6 +114,7 @@ public class ControlService extends Service { final HtmlRedirect redirectToActors = new HtmlRedirect("/actors"); final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys"); final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage"); + final HtmlRedirect redirectToBlacklist = new HtmlRedirect("/blacklist"); final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints"); final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue"); @@ -145,6 +151,11 @@ public class ControlService extends Service { Spark.post("/public/storage/specs", controlActorService::createCrawlSpecification, redirectToStorage); Spark.post("/public/storage/:fid/delete", controlFileStorageService::flagFileForDeletionRequest, redirectToStorage); + // Blacklist + + Spark.get("/public/blacklist", this::blacklistModel, blacklistRenderer::render); + Spark.post("/public/blacklist", this::updateBlacklist, redirectToBlacklist); + // API Keys Spark.get("/public/api-keys", this::apiKeysModel, apiKeysRenderer::render); @@ -171,6 +182,22 @@ public class ControlService extends Service { monitors.subscribe(this::logMonitorStateChange); } + private Object blacklistModel(Request request, Response response) { + return Map.of("blacklist", blacklistService.lastNAdditions(100)); + } + + private Object updateBlacklist(Request request, Response response) { + var domain = new EdgeDomain(request.queryParams("domain")); + if ("add".equals(request.queryParams("act"))) { + var comment = Objects.requireNonNullElse(request.queryParams("comment"), ""); + blacklistService.addToBlacklist(domain, comment); + } else if ("del".equals(request.queryParams("act"))) { + blacklistService.removeFromBlacklist(domain); + } + + return ""; + } + private Object overviewModel(Request request, Response response) { return Map.of("processes", heartbeatService.getProcessHeartbeats(), diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java new file mode 100644 index 00000000..e7db4805 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java @@ -0,0 +1,6 @@ +package nu.marginalia.control.model; + +import nu.marginalia.model.EdgeDomain; + +public record BlacklistedDomainModel(EdgeDomain domain, String comment) { +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java new file mode 100644 index 00000000..d23a06e2 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java @@ -0,0 +1,79 @@ +package nu.marginalia.control.svc; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.control.model.BlacklistedDomainModel; +import nu.marginalia.model.EdgeDomain; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class ControlBlacklistService { + + private final HikariDataSource dataSource; + + @Inject + public ControlBlacklistService(HikariDataSource dataSource) { + this.dataSource = dataSource; + } + + public void addToBlacklist(EdgeDomain domain, String comment) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN, COMMENT) VALUES (?, ?) + """)) { + stmt.setString(1, domain.toString()); + stmt.setString(2, comment); + stmt.executeUpdate(); + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + public void removeFromBlacklist(EdgeDomain domain) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=? + """)) { + stmt.setString(1, domain.toString()); + stmt.addBatch(); + stmt.setString(1, domain.domain); + stmt.addBatch(); + stmt.executeBatch(); + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + public List lastNAdditions(int n) { + final List ret = new ArrayList<>(n); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT URL_DOMAIN, COMMENT + FROM EC_DOMAIN_BLACKLIST + ORDER BY ID DESC + LIMIT ? + """)) { + stmt.setInt(1, n); + + var rs = stmt.executeQuery(); + while (rs.next()) { + ret.add(new BlacklistedDomainModel( + new EdgeDomain(rs.getString("URL_DOMAIN")), + rs.getString("COMMENT") + ) + ); + } + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + + return ret; + + } +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java index bf36bfad..758d0313 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java @@ -18,10 +18,14 @@ import java.util.Optional; */ public class DomainComplaintService { private final HikariDataSource dataSource; + private final ControlBlacklistService blacklistService; @Inject - public DomainComplaintService(HikariDataSource dataSource) { + public DomainComplaintService(HikariDataSource dataSource, + ControlBlacklistService blacklistService + ) { this.dataSource = dataSource; + this.blacklistService = blacklistService; } public List getComplaints() { @@ -53,12 +57,13 @@ public class DomainComplaintService { } public void approveAppealBlacklisting(EdgeDomain domain) { - removeFromBlacklist(domain); + blacklistService.removeFromBlacklist(domain); setDecision(domain, "APPROVED"); } public void blacklistDomain(EdgeDomain domain) { - addToBlacklist(domain); + blacklistService.addToBlacklist(domain, "Domain complaint"); + setDecision(domain, "BLACKLISTED"); } @@ -66,33 +71,7 @@ public class DomainComplaintService { setDecision(domain, "REJECTED"); } - private void addToBlacklist(EdgeDomain domain) { - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement(""" - INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN) VALUES (?) - """)) { - stmt.setString(1, domain.toString()); - stmt.executeUpdate(); - } - catch (SQLException ex) { - throw new RuntimeException(ex); - } - } - private void removeFromBlacklist(EdgeDomain domain) { - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement(""" - DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=? - """)) { - stmt.setString(1, domain.toString()); - stmt.addBatch(); - stmt.setString(1, domain.domain); - stmt.executeBatch(); - } - catch (SQLException ex) { - throw new RuntimeException(ex); - } - } private void setDecision(EdgeDomain domain, String decision) { try (var conn = dataSource.getConnection(); diff --git a/code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb b/code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb new file mode 100644 index 00000000..5622659c --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb @@ -0,0 +1,68 @@ + + + + Control Service + + + + +{{> control/partials/nav}} +
+

Blacklist

+ +

+ The blacklist is a list of sanctioned domains that will not be + crawled, indexed, or returned from the search results. +

+ + + + + + + + + + + + + +
DescriptionAction
Add To Blacklist

+ This will add the given domain to the blacklist. +

+
+  
+   +
+
+ +
+
Remove from blacklist

+ Remove the specified domain from the blacklist. This will ensure that + the domain is not blacklisted, in doing so it may remove the root domain + from the blacklist as well. +

+
+   +
+
+ +
+
+ +

Recent Additions

+ + + + + + {{#each blacklist}} + + + + + {{/each}} +
DomainComment
{{domain}}{{comment}}
+
+ + \ No newline at end of file