Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(converter) GUI for dealing with user complaints

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-03 17:59:57 +02:00
parent f01f608474
commit 1d0cea1d55
9 changed files with 349 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java
View File

@ -4,9 +4,12 @@ import com.google.gson.Gson;
import com.google.inject.Inject;
import nu.marginalia.client.ServiceMonitors;
import nu.marginalia.control.model.Actor;
import nu.marginalia.control.model.DomainComplaintModel;
import nu.marginalia.control.model.ProcessHeartbeat;
import nu.marginalia.control.svc.*;
import nu.marginalia.db.storage.model.FileStorageId;
import nu.marginalia.db.storage.model.FileStorageType;
import nu.marginalia.model.EdgeDomain;
import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.mq.MqMessageState;
import nu.marginalia.mq.persistence.MqPersistence;
@ -21,7 +24,10 @@ import spark.Spark;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class ControlService extends Service {
@ -32,6 +38,7 @@ public class ControlService extends Service {
private final HeartbeatService heartbeatService;
private final EventLogService eventLogService;
private final ApiKeyService apiKeyService;
private final DomainComplaintService domainComplaintService;
private final ControlActorService controlActorService;
private final StaticResources staticResources;
private final MessageQueueViewService messageQueueViewService;
@ -49,6 +56,7 @@ public class ControlService extends Service {
MessageQueueViewService messageQueueViewService,
ControlFileStorageService controlFileStorageService,
ApiKeyService apiKeyService,
DomainComplaintService domainComplaintService,
MqPersistence persistence
) throws IOException {
@ -57,6 +65,7 @@ public class ControlService extends Service {
this.heartbeatService = heartbeatService;
this.eventLogService = eventLogService;
this.apiKeyService = apiKeyService;
this.domainComplaintService = domainComplaintService;
var indexRenderer = rendererFactory.renderer("control/index");
var servicesRenderer = rendererFactory.renderer("control/services");
@ -69,6 +78,7 @@ public class ControlService extends Service {
var storageProcessedRenderer = rendererFactory.renderer("control/storage-processed");
var apiKeysRenderer = rendererFactory.renderer("control/api-keys");
var domainComplaintsRenderer = rendererFactory.renderer("control/domain-complaints");
var storageDetailsRenderer = rendererFactory.renderer("control/storage-details");
var updateMessageStateRenderer = rendererFactory.renderer("control/dialog-update-message-state");
@ -103,6 +113,7 @@ public class ControlService extends Service {
final HtmlRedirect redirectToProcesses = new HtmlRedirect("/actors");
final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys");
final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage");
final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints");
Spark.post("/public/fsms/:fsm/start", controlActorService::startFsm, redirectToProcesses);
Spark.post("/public/fsms/:fsm/stop", controlActorService::stopFsm, redirectToProcesses);
@ -120,6 +131,9 @@ public class ControlService extends Service {
// HTML forms don't support the DELETE verb :-(
Spark.post("/public/api-keys/:key/delete", this::deleteApiKey, redirectToApiKeys);
Spark.get("/public/complaints", this::complaintsModel, domainComplaintsRenderer::render);
Spark.post("/public/complaints/:domain", this::reviewComplaint, redirectToComplaints);
Spark.get("/public/message/:id/state", (rq, rsp) -> persistence.getMessage(Long.parseLong(rq.params("id"))), updateMessageStateRenderer::render);
Spark.post("/public/message/:id/state", (rq, rsp) -> {
MqMessageState state = MqMessageState.valueOf(rq.queryParams("state"));
@ -133,6 +147,35 @@ public class ControlService extends Service {
monitors.subscribe(this::logMonitorStateChange);
}
private Object complaintsModel(Request request, Response response) {
Map<Boolean, List<DomainComplaintModel>> complaintsByReviewed =
domainComplaintService.getComplaints().stream().collect(Collectors.partitioningBy(DomainComplaintModel::reviewed));
var reviewed = complaintsByReviewed.get(true);
var unreviewed = complaintsByReviewed.get(false);
reviewed.sort(Comparator.comparing(DomainComplaintModel::reviewDate).reversed());
unreviewed.sort(Comparator.comparing(DomainComplaintModel::fileDate).reversed());
return Map.of("complaintsNew", unreviewed, "complaintsReviewed", reviewed);
}
private Object reviewComplaint(Request request, Response response) {
var domain = new EdgeDomain(request.params("domain"));
String action = request.queryParams("action");
logger.info("Reviewing complaint for domain {} with action {}", domain, action);
switch (action) {
case "noop" -> domainComplaintService.reviewNoAction(domain);
case "appeal" -> domainComplaintService.approveAppealBlacklisting(domain);
case "blacklist" -> domainComplaintService.blacklistDomain(domain);
default -> throw new UnsupportedOperationException();
}
return "";
}
private Object createApiKey(Request request, Response response) {
String license = request.queryParams("license");
String name = request.queryParams("name");
@ -223,7 +266,11 @@ public class ControlService extends Service {
}
private Object processesModel(Request request, Response response) {
return Map.of("processes", heartbeatService.getProcessHeartbeats(),
var heartbeatsAll = heartbeatService.getProcessHeartbeats();
var byIsJob = heartbeatsAll.stream().collect(Collectors.partitioningBy(ProcessHeartbeat::isServiceJob));
return Map.of("processes", byIsJob.get(false),
"jobs", byIsJob.get(true),
"actors", controlActorService.getActorStates(),
"messages", messageQueueViewService.getLastEntries(20));
}

28
code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintCategory.java Normal file
View File

@ -0,0 +1,28 @@
package nu.marginalia.control.model;
public enum DomainComplaintCategory {
SPAM("spam"),
FREEBOOTING("freebooting"),
BROKEN("broken"),
SHOCK("shock"),
BLACKLIST("blacklist"),
UNKNOWN("unknown");
private final String categoryName;
DomainComplaintCategory(String categoryName) {
this.categoryName = categoryName;
}
public String categoryName() {
return categoryName;
}
public static DomainComplaintCategory fromCategoryName(String categoryName) {
for (DomainComplaintCategory category : values()) {
if (category.categoryName().equals(categoryName)) {
return category;
}
}
return UNKNOWN;
}
}

17
code/services-core/control-service/src/main/java/nu/marginalia/control/model/DomainComplaintModel.java Normal file
View File

@ -0,0 +1,17 @@
package nu.marginalia.control.model;
public record DomainComplaintModel(String domain,
DomainComplaintCategory category,
String description,
String sample,
String decision,
String fileDate,
String reviewDate,
boolean reviewed)
{
public boolean isAppeal() {
return category == DomainComplaintCategory.BLACKLIST;
}
}

112
code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java Normal file
View File

@ -0,0 +1,112 @@
package nu.marginalia.control.svc;
import com.google.inject.Inject;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.control.model.DomainComplaintCategory;
import nu.marginalia.control.model.DomainComplaintModel;
import nu.marginalia.model.EdgeDomain;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
/** Service for handling domain complaints. This code has an user-facing correspondent in
* SearchFlagSiteService in search-service
*/
public class DomainComplaintService {
private final HikariDataSource dataSource;
@Inject
public DomainComplaintService(HikariDataSource dataSource) {
this.dataSource = dataSource;
}
public List<DomainComplaintModel> getComplaints() {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT EC_DOMAIN.DOMAIN_NAME AS DOMAIN, CATEGORY, DESCRIPTION, SAMPLE, FILE_DATE, REVIEWED, DECISION, REVIEW_DATE
FROM DOMAIN_COMPLAINT LEFT JOIN EC_DOMAIN ON EC_DOMAIN.ID=DOMAIN_COMPLAINT.DOMAIN_ID
""")) {
List<DomainComplaintModel> complaints = new ArrayList<>();
var rs = stmt.executeQuery();
while (rs.next()) {
complaints.add(new DomainComplaintModel(
rs.getString("DOMAIN"),
DomainComplaintCategory.fromCategoryName(rs.getString("CATEGORY")),
rs.getString("DESCRIPTION"),
rs.getString("SAMPLE"),
rs.getString("DECISION"),
rs.getTimestamp("FILE_DATE").toLocalDateTime().toString(),
Optional.ofNullable(rs.getTimestamp("REVIEW_DATE"))
.map(Timestamp::toLocalDateTime).map(Object::toString).orElse(null),
rs.getBoolean("REVIEWED")
));
}
return complaints;
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
public void approveAppealBlacklisting(EdgeDomain domain) {
removeFromBlacklist(domain);
setDecision(domain, "APPROVED");
}
public void blacklistDomain(EdgeDomain domain) {
addToBlacklist(domain);
setDecision(domain, "BLACKLISTED");
}
public void reviewNoAction(EdgeDomain domain) {
setDecision(domain, "REJECTED");
}
private void addToBlacklist(EdgeDomain domain) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN) VALUES (?)
""")) {
stmt.setString(1, domain.toString());
stmt.executeUpdate();
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
private void removeFromBlacklist(EdgeDomain domain) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?
""")) {
stmt.setString(1, domain.toString());
stmt.addBatch();
stmt.setString(1, domain.domain);
stmt.executeBatch();
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
private void setDecision(EdgeDomain domain, String decision) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
UPDATE DOMAIN_COMPLAINT SET DECISION=?, REVIEW_DATE=NOW()
WHERE DOMAIN_ID=(SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=?)
AND DECISION IS NULL
""")) {
stmt.setString(1, decision);
stmt.setString(2, domain.toString());
stmt.executeUpdate();
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
}

25
code/services-core/control-service/src/main/resources/static/control/style.css
View File

@ -49,9 +49,34 @@ table {
}
th { text-align: left; }
td,th { padding-right: 1ch; border: 1px solid #ccc; }
tr:nth-of-type(2n) {
background-color: #eee;
}
table.table-rh-2 tr:nth-of-type(4n+1) { background-color: #eee; }
table.table-rh-2 tr:nth-of-type(4n+2) { background-color: #eee; }
table.table-rh-2 tr:nth-of-type(4n+3) { background-color: unset; }
table.table-rh-2 tr:nth-of-type(4n) { background-color: unset; }
table.table-rh-2 tr:nth-of-type(4n) td,
table.table-rh-2 tr:nth-of-type(4n) th { border-bottom: 1px solid #888; }
table.table-rh-2 tr:nth-of-type(4n+2) td,
table.table-rh-2 tr:nth-of-type(4n+2) th { border-bottom: 1px solid #888; }
table.table-rh-3 tr:nth-of-type(6n+1) { background-color: #eee; }
table.table-rh-3 tr:nth-of-type(6n+2) { background-color: #eee; }
table.table-rh-3 tr:nth-of-type(6n+3) { background-color: #eee; }
table.table-rh-3 tr:nth-of-type(6n+4) { background-color: unset; }
table.table-rh-3 tr:nth-of-type(6n+5) { background-color: unset; }
table.table-rh-3 tr:nth-of-type(6n) { background-color: unset; }
table.table-rh-3 tr:nth-of-type(6n) td,
table.table-rh-3 tr:nth-of-type(6n) th { border-bottom: 1px solid #888; }
table.table-rh-3 tr:nth-of-type(6n+3) td,
table.table-rh-3 tr:nth-of-type(6n+3) th { border-bottom: 1px solid #888; }
body > nav {
grid-area: left;
}

2
code/services-core/control-service/src/main/resources/templates/control/api-keys.hdb
View File

@ -10,7 +10,7 @@
<section>
<h1>API Keys</h1>
<table id="apikeys">
<table id="apikeys" class="table-rh-2">
<tr>
<th colspan="3">Key</th>
<th>&nbsp;</th>

111
code/services-core/control-service/src/main/resources/templates/control/domain-complaints.hdb Normal file
View File

@ -0,0 +1,111 @@
<!DOCTYPE html>
<html>
<head>
<title>Control Service</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="/style.css" />
</head>
<body>
{{> control/partials/nav}}
<section>
<!-- public record DomainComplaintModel(String domain,
DomainComplaintCategory category,
String description,
String sample,
String decision,
String reviewDate,
boolean reviewed) -->
<h1>Domain Complaints</h1>
{{#unless complaintsNew}}
<p>No new complaints!</p>
{{/unless}}
{{#if complaintsNew}}
<table class="table-rh-3" id="complaintsNew">
<tr>
<th>Date</th>
<th>Category</th>
<th></th>
</tr>
<tr>
<th>Domain</th>
<th colspan="2">Sample</th>
</tr>
<tr>
<th colspan="3">Description</th>
</tr>
{{#each complaintsNew}}
<tr>
<td>{{fileDate}}</td>
<td>{{category}}</td>
<td>
<form method="post" action="/complaints/{{domain}}" onsubmit="return confirm('Confirm review of {{domain}}')">
<label for="action-{{domain}}" style="display: none;">Action: </label>
<select type="select" name="action" id="action-{{domain}}">
{{#if appeal}}
<option value="appeal">Revert Ban</option>
{{/if}}
{{#unless appeal}}
<option value="blacklist">Ban Domain</option>
{{/unless}}
<option selected value="noop">No Action</option>
</select>
<input type="submit" value="Review" />
</form>
</td>
</tr>
<tr>
<td><a href="https://search.marginalia.nu/site/{{domain}}">{{domain}}</a></td>
<td colspan="2">{{sample}}</td>
</tr>
<tr>
<td colspan="3">{{description}}</td>
</tr>
{{/each}}
</table>
{{/if}}
{{#if complaintsReviewed}}
<h1>Review Log</h1>
<table class="table-rh-3" id="complaintsReviewed">
<tr>
<th>Review Date</th>
<th>Category</th>
<th>Action</th>
</tr>
<tr>
<th>Domain</th>
<th colspan="2">Sample</th>
</tr>
<tr>
<th colspan="3">Description</th>
</tr>
{{#each complaintsReviewed}}
<tr>
<td>{{fileDate}}</td>
<td>{{category}}</td>
<td>
{{decision}}
</td>
</tr>
<tr>
<td><a href="https://search.marginalia.nu/site/{{domain}}">{{domain}}</a></td>
<td colspan="2">{{sample}}</td>
</tr>
<tr>
<td colspan="3">{{description}}</td>
</tr>
{{/each}}
</table>
{{/if}}
</section>
</body>
<script src="/refresh.js"></script>
<script>
window.setInterval(() => {
refresh([]);
}, 2000);
</script>
</html>

3
code/services-core/control-service/src/main/resources/templates/control/partials/message-queue-table.hdb
View File

@ -1,6 +1,6 @@
<h1>Message Queue</h1>
<table id="queue">
<table id="queue" class="table-rh-2">
<tr>
<th>State<br>TTL</th>
<th>Msg ID<br>Related ID</th>
@ -9,6 +9,7 @@
<th>Owner Instance<br>Owner Tick</th>
<th>Created<br>Updated</th>
</tr>
<tr></tr>
{{#each messages}}
<tr>
<td>{{stateCode}}&nbsp;<a onClick="updateMsgState({{id}})" href="/message/{{id}}/state">{{state}}</a></td>

3
code/services-core/search-service/src/main/java/nu/marginalia/search/svc/SearchFlagSiteService.java
View File

@ -17,6 +17,9 @@ import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
/** Service for handling flagging sites. This code has an admin-facing correspondent in
* DomainComplaintService in control-service
*/
public class SearchFlagSiteService {
private final MustacheRenderer<FlagSiteViewModel> formTemplate;
private final HikariDataSource dataSource;