diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java index c62631ef..1ec0e555 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java @@ -4,9 +4,12 @@ import com.google.gson.Gson; import com.google.inject.Inject; import nu.marginalia.client.ServiceMonitors; import nu.marginalia.control.model.Actor; +import nu.marginalia.control.model.DomainComplaintModel; +import nu.marginalia.control.model.ProcessHeartbeat; import nu.marginalia.control.svc.*; import nu.marginalia.db.storage.model.FileStorageId; import nu.marginalia.db.storage.model.FileStorageType; +import nu.marginalia.model.EdgeDomain; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.mq.MqMessageState; import nu.marginalia.mq.persistence.MqPersistence; @@ -21,7 +24,10 @@ import spark.Spark; import java.io.IOException; import java.sql.SQLException; +import java.util.Comparator; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class ControlService extends Service { @@ -32,6 +38,7 @@ public class ControlService extends Service { private final HeartbeatService heartbeatService; private final EventLogService eventLogService; private final ApiKeyService apiKeyService; + private final DomainComplaintService domainComplaintService; private final ControlActorService controlActorService; private final StaticResources staticResources; private final MessageQueueViewService messageQueueViewService; @@ -49,6 +56,7 @@ public class ControlService extends Service { MessageQueueViewService messageQueueViewService, ControlFileStorageService controlFileStorageService, ApiKeyService apiKeyService, + DomainComplaintService domainComplaintService, MqPersistence persistence ) throws IOException { @@ -57,6 +65,7 @@ public class ControlService extends Service { this.heartbeatService = heartbeatService; this.eventLogService = eventLogService; this.apiKeyService = apiKeyService; + this.domainComplaintService = domainComplaintService; var indexRenderer = rendererFactory.renderer("control/index"); var servicesRenderer = rendererFactory.renderer("control/services"); @@ -69,6 +78,7 @@ public class ControlService extends Service { var storageProcessedRenderer = rendererFactory.renderer("control/storage-processed"); var apiKeysRenderer = rendererFactory.renderer("control/api-keys"); + var domainComplaintsRenderer = rendererFactory.renderer("control/domain-complaints"); var storageDetailsRenderer = rendererFactory.renderer("control/storage-details"); var updateMessageStateRenderer = rendererFactory.renderer("control/dialog-update-message-state"); @@ -103,6 +113,7 @@ public class ControlService extends Service { final HtmlRedirect redirectToProcesses = new HtmlRedirect("/actors"); final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys"); final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage"); + final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints"); Spark.post("/public/fsms/:fsm/start", controlActorService::startFsm, redirectToProcesses); Spark.post("/public/fsms/:fsm/stop", controlActorService::stopFsm, redirectToProcesses); @@ -120,6 +131,9 @@ public class ControlService extends Service { // HTML forms don't support the DELETE verb :-( Spark.post("/public/api-keys/:key/delete", this::deleteApiKey, redirectToApiKeys); + Spark.get("/public/complaints", this::complaintsModel, domainComplaintsRenderer::render); + Spark.post("/public/complaints/:domain", this::reviewComplaint, redirectToComplaints); + Spark.get("/public/message/:id/state", (rq, rsp) -> persistence.getMessage(Long.parseLong(rq.params("id"))), updateMessageStateRenderer::render); Spark.post("/public/message/:id/state", (rq, rsp) -> { MqMessageState state = MqMessageState.valueOf(rq.queryParams("state")); @@ -133,6 +147,35 @@ public class ControlService extends Service { monitors.subscribe(this::logMonitorStateChange); } + private Object complaintsModel(Request request, Response response) { + Map> complaintsByReviewed = + domainComplaintService.getComplaints().stream().collect(Collectors.partitioningBy(DomainComplaintModel::reviewed)); + + var reviewed = complaintsByReviewed.get(true); + var unreviewed = complaintsByReviewed.get(false); + + reviewed.sort(Comparator.comparing(DomainComplaintModel::reviewDate).reversed()); + unreviewed.sort(Comparator.comparing(DomainComplaintModel::fileDate).reversed()); + + return Map.of("complaintsNew", unreviewed, "complaintsReviewed", reviewed); + } + + private Object reviewComplaint(Request request, Response response) { + var domain = new EdgeDomain(request.params("domain")); + String action = request.queryParams("action"); + + logger.info("Reviewing complaint for domain {} with action {}", domain, action); + + switch (action) { + case "noop" -> domainComplaintService.reviewNoAction(domain); + case "appeal" -> domainComplaintService.approveAppealBlacklisting(domain); + case "blacklist" -> domainComplaintService.blacklistDomain(domain); + default -> throw new UnsupportedOperationException(); + } + + return ""; + } + private Object createApiKey(Request request, Response response) { String license = request.queryParams("license"); String name = request.queryParams("name"); @@ -223,7 +266,11 @@ public class ControlService extends Service { } private Object processesModel(Request request, Response response) { - return Map.of("processes", heartbeatService.getProcessHeartbeats(), + var heartbeatsAll = heartbeatService.getProcessHeartbeats(); + var byIsJob = heartbeatsAll.stream().collect(Collectors.partitioningBy(ProcessHeartbeat::isServiceJob)); + + return Map.of("processes", byIsJob.get(false), + "jobs", byIsJob.get(true), "actors", controlActorService.getActorStates(), "messages", messageQueueViewService.getLastEntries(20)); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintCategory.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintCategory.java new file mode 100644 index 00000000..d1743ba9 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintCategory.java @@ -0,0 +1,28 @@ +package nu.marginalia.control.model; + +public enum DomainComplaintCategory { + SPAM("spam"), + FREEBOOTING("freebooting"), + BROKEN("broken"), + SHOCK("shock"), + BLACKLIST("blacklist"), + UNKNOWN("unknown"); + + private final String categoryName; + + DomainComplaintCategory(String categoryName) { + this.categoryName = categoryName; + } + + public String categoryName() { + return categoryName; + } + public static DomainComplaintCategory fromCategoryName(String categoryName) { + for (DomainComplaintCategory category : values()) { + if (category.categoryName().equals(categoryName)) { + return category; + } + } + return UNKNOWN; + } +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintModel.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintModel.java new file mode 100644 index 00000000..603b6fc8 --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintModel.java @@ -0,0 +1,17 @@ +package nu.marginalia.control.model; + +public record DomainComplaintModel(String domain, + DomainComplaintCategory category, + String description, + String sample, + String decision, + String fileDate, + String reviewDate, + boolean reviewed) +{ + + public boolean isAppeal() { + return category == DomainComplaintCategory.BLACKLIST; + } + +} diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java new file mode 100644 index 00000000..bf36bfad --- /dev/null +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java @@ -0,0 +1,112 @@ +package nu.marginalia.control.svc; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.control.model.DomainComplaintCategory; +import nu.marginalia.control.model.DomainComplaintModel; +import nu.marginalia.model.EdgeDomain; + +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + + +/** Service for handling domain complaints. This code has an user-facing correspondent in + * SearchFlagSiteService in search-service + */ +public class DomainComplaintService { + private final HikariDataSource dataSource; + + @Inject + public DomainComplaintService(HikariDataSource dataSource) { + this.dataSource = dataSource; + } + + public List getComplaints() { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT EC_DOMAIN.DOMAIN_NAME AS DOMAIN, CATEGORY, DESCRIPTION, SAMPLE, FILE_DATE, REVIEWED, DECISION, REVIEW_DATE + FROM DOMAIN_COMPLAINT LEFT JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_COMPLAINT.DOMAIN_ID + """)) { + List complaints = new ArrayList<>(); + var rs = stmt.executeQuery(); + while (rs.next()) { + complaints.add(new DomainComplaintModel( + rs.getString("DOMAIN"), + DomainComplaintCategory.fromCategoryName(rs.getString("CATEGORY")), + rs.getString("DESCRIPTION"), + rs.getString("SAMPLE"), + rs.getString("DECISION"), + rs.getTimestamp("FILE_DATE").toLocalDateTime().toString(), + Optional.ofNullable(rs.getTimestamp("REVIEW_DATE")) + .map(Timestamp::toLocalDateTime).map(Object::toString).orElse(null), + rs.getBoolean("REVIEWED") + )); + } + return complaints; + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + public void approveAppealBlacklisting(EdgeDomain domain) { + removeFromBlacklist(domain); + setDecision(domain, "APPROVED"); + } + + public void blacklistDomain(EdgeDomain domain) { + addToBlacklist(domain); + setDecision(domain, "BLACKLISTED"); + } + + public void reviewNoAction(EdgeDomain domain) { + setDecision(domain, "REJECTED"); + } + + private void addToBlacklist(EdgeDomain domain) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN) VALUES (?) + """)) { + stmt.setString(1, domain.toString()); + stmt.executeUpdate(); + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + private void removeFromBlacklist(EdgeDomain domain) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=? + """)) { + stmt.setString(1, domain.toString()); + stmt.addBatch(); + stmt.setString(1, domain.domain); + stmt.executeBatch(); + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + private void setDecision(EdgeDomain domain, String decision) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + UPDATE DOMAIN_COMPLAINT SET DECISION=?, REVIEW_DATE=NOW() + WHERE DOMAIN_ID=(SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?) + AND DECISION IS NULL + """)) { + stmt.setString(1, decision); + stmt.setString(2, domain.toString()); + stmt.executeUpdate(); + } + catch (SQLException ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/code/services-core/control-service/src/main/resources/static/control/style.css b/code/services-core/control-service/src/main/resources/static/control/style.css index e3722cb4..a248a499 100644 --- a/code/services-core/control-service/src/main/resources/static/control/style.css +++ b/code/services-core/control-service/src/main/resources/static/control/style.css @@ -49,9 +49,34 @@ table { } th { text-align: left; } td,th { padding-right: 1ch; border: 1px solid #ccc; } + tr:nth-of-type(2n) { background-color: #eee; } + + +table.table-rh-2 tr:nth-of-type(4n+1) { background-color: #eee; } +table.table-rh-2 tr:nth-of-type(4n+2) { background-color: #eee; } +table.table-rh-2 tr:nth-of-type(4n+3) { background-color: unset; } +table.table-rh-2 tr:nth-of-type(4n) { background-color: unset; } + +table.table-rh-2 tr:nth-of-type(4n) td, +table.table-rh-2 tr:nth-of-type(4n) th { border-bottom: 1px solid #888; } +table.table-rh-2 tr:nth-of-type(4n+2) td, +table.table-rh-2 tr:nth-of-type(4n+2) th { border-bottom: 1px solid #888; } + +table.table-rh-3 tr:nth-of-type(6n+1) { background-color: #eee; } +table.table-rh-3 tr:nth-of-type(6n+2) { background-color: #eee; } +table.table-rh-3 tr:nth-of-type(6n+3) { background-color: #eee; } +table.table-rh-3 tr:nth-of-type(6n+4) { background-color: unset; } +table.table-rh-3 tr:nth-of-type(6n+5) { background-color: unset; } +table.table-rh-3 tr:nth-of-type(6n) { background-color: unset; } + +table.table-rh-3 tr:nth-of-type(6n) td, +table.table-rh-3 tr:nth-of-type(6n) th { border-bottom: 1px solid #888; } +table.table-rh-3 tr:nth-of-type(6n+3) td, +table.table-rh-3 tr:nth-of-type(6n+3) th { border-bottom: 1px solid #888; } + body > nav { grid-area: left; } diff --git a/code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb b/code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb index 5361fb82..e58b6b8a 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb @@ -10,7 +10,7 @@

API Keys

- +
diff --git a/code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb b/code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb new file mode 100644 index 00000000..ac1f6c88 --- /dev/null +++ b/code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb @@ -0,0 +1,111 @@ + + + + Control Service + + + + +{{> control/partials/nav}} +
+ +

Domain Complaints

+ {{#unless complaintsNew}} +

No new complaints!

+ {{/unless}} + {{#if complaintsNew}} +
Key  
+ + + + + + + + + + + + + + {{#each complaintsNew}} + + + + + + + + + + + + + + {{/each}} +
DateCategory
DomainSample
Description
{{fileDate}}{{category}} +
+ + + +
+
{{domain}}{{sample}}
{{description}}
+ {{/if}} + + {{#if complaintsReviewed}} +

Review Log

+ + + + + + + + + + + + + + + {{#each complaintsReviewed}} + + + + + + + + + + + + + {{/each}} +
Review DateCategoryAction
DomainSample
Description
{{fileDate}}{{category}} + {{decision}} +
{{domain}}{{sample}}
{{description}}
+ {{/if}} +
+ + + + \ No newline at end of file diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb b/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb index cc8d98a2..b971c928 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb @@ -1,6 +1,6 @@

Message Queue

- +
@@ -9,6 +9,7 @@ + {{#each messages}} diff --git a/code/services-core/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java b/code/services-core/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java index 5eb960a5..33d0165d 100644 --- a/code/services-core/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java +++ b/code/services-core/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java @@ -17,6 +17,9 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +/** Service for handling flagging sites. This code has an admin-facing correspondent in + * DomainComplaintService in control-service + */ public class SearchFlagSiteService { private final MustacheRenderer formTemplate; private final HikariDataSource dataSource; @@ -83,9 +86,9 @@ public class SearchFlagSiteService { try (var conn = dataSource.getConnection(); var complaintsStmt = conn.prepareStatement(""" - SELECT CATEGORY, FILE_DATE, REVIEWED, DECISION + SELECT CATEGORY, FILE_DATE, REVIEWED, DECISION FROM DOMAIN_COMPLAINT - WHERE DOMAIN_ID=? + WHERE DOMAIN_ID=? """); var stmt = conn.prepareStatement( """
State
TTL		 Msg ID
Related ID		Owner Instance
Owner Tick		 Created
Updated
{{stateCode}} {{state}}