Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(blacklist) Blacklist management

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-10 15:40:07 +02:00
parent 807fb2d052
commit 4f8048be31
6 changed files with 189 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
code/common/db/src/main/resources/db/migration/V23_06_0_001__blacklist.sql
View File

@ -2,6 +2,7 @@
CREATE TABLE IF NOT EXISTS EC_DOMAIN_BLACKLIST (
ID INT PRIMARY KEY AUTO_INCREMENT,
URL_DOMAIN VARCHAR(255) UNIQUE NOT NULL
COMMENT VARCHAR(255) DEFAULT NULL
)
CHARACTER SET utf8mb4
COLLATE utf8mb4_unicode_ci;

27
code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java
View File

@ -24,6 +24,7 @@ import java.sql.SQLException;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
public class ControlService extends Service {
@ -36,6 +37,7 @@ public class ControlService extends Service {
private final EventLogService eventLogService;
private final ApiKeyService apiKeyService;
private final DomainComplaintService domainComplaintService;
private final ControlBlacklistService blacklistService;
private final ControlActorService controlActorService;
private final StaticResources staticResources;
private final MessageQueueService messageQueueService;
@ -54,6 +56,7 @@ public class ControlService extends Service {
ControlFileStorageService controlFileStorageService,
ApiKeyService apiKeyService,
DomainComplaintService domainComplaintService,
ControlBlacklistService blacklistService,
ControlActionsService controlActionsService
) throws IOException {
@ -63,6 +66,7 @@ public class ControlService extends Service {
this.eventLogService = eventLogService;
this.apiKeyService = apiKeyService;
this.domainComplaintService = domainComplaintService;
this.blacklistService = blacklistService;
var indexRenderer = rendererFactory.renderer("control/index");
var servicesRenderer = rendererFactory.renderer("control/services");
@ -85,6 +89,7 @@ public class ControlService extends Service {
var viewMessageRenderer = rendererFactory.renderer("control/view-message");
var actionsViewRenderer = rendererFactory.renderer("control/actions");
var blacklistRenderer = rendererFactory.renderer("control/blacklist");
this.controlActorService = controlActorService;
@ -109,6 +114,7 @@ public class ControlService extends Service {
final HtmlRedirect redirectToActors = new HtmlRedirect("/actors");
final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys");
final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage");
final HtmlRedirect redirectToBlacklist = new HtmlRedirect("/blacklist");
final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints");
final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue");
@ -145,6 +151,11 @@ public class ControlService extends Service {
Spark.post("/public/storage/specs", controlActorService::createCrawlSpecification, redirectToStorage);
Spark.post("/public/storage/:fid/delete", controlFileStorageService::flagFileForDeletionRequest, redirectToStorage);
// Blacklist
Spark.get("/public/blacklist", this::blacklistModel, blacklistRenderer::render);
Spark.post("/public/blacklist", this::updateBlacklist, redirectToBlacklist);
// API Keys
Spark.get("/public/api-keys", this::apiKeysModel, apiKeysRenderer::render);
@ -171,6 +182,22 @@ public class ControlService extends Service {
monitors.subscribe(this::logMonitorStateChange);
}
private Object blacklistModel(Request request, Response response) {
return Map.of("blacklist", blacklistService.lastNAdditions(100));
}
private Object updateBlacklist(Request request, Response response) {
var domain = new EdgeDomain(request.queryParams("domain"));
if ("add".equals(request.queryParams("act"))) {
var comment = Objects.requireNonNullElse(request.queryParams("comment"), "");
blacklistService.addToBlacklist(domain, comment);
} else if ("del".equals(request.queryParams("act"))) {
blacklistService.removeFromBlacklist(domain);
}
return "";
}
private Object overviewModel(Request request, Response response) {
return Map.of("processes", heartbeatService.getProcessHeartbeats(),

6
code/services-core/control-service/src/main/java/nu/marginalia/control/model/BlacklistedDomainModel.java Normal file
View File

@ -0,0 +1,6 @@
package nu.marginalia.control.model;
import nu.marginalia.model.EdgeDomain;
public record BlacklistedDomainModel(EdgeDomain domain, String comment) {
}

79
code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java Normal file
View File

@ -0,0 +1,79 @@
package nu.marginalia.control.svc;
import com.google.inject.Inject;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.control.model.BlacklistedDomainModel;
import nu.marginalia.model.EdgeDomain;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
public class ControlBlacklistService {
private final HikariDataSource dataSource;
@Inject
public ControlBlacklistService(HikariDataSource dataSource) {
this.dataSource = dataSource;
}
public void addToBlacklist(EdgeDomain domain, String comment) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN, COMMENT) VALUES (?, ?)
""")) {
stmt.setString(1, domain.toString());
stmt.setString(2, comment);
stmt.executeUpdate();
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
public void removeFromBlacklist(EdgeDomain domain) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?
""")) {
stmt.setString(1, domain.toString());
stmt.addBatch();
stmt.setString(1, domain.domain);
stmt.addBatch();
stmt.executeBatch();
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
public List<BlacklistedDomainModel> lastNAdditions(int n) {
final List<BlacklistedDomainModel> ret = new ArrayList<>(n);
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT URL_DOMAIN, COMMENT
FROM EC_DOMAIN_BLACKLIST
ORDER BY ID DESC
LIMIT ?
""")) {
stmt.setInt(1, n);
var rs = stmt.executeQuery();
while (rs.next()) {
ret.add(new BlacklistedDomainModel(
new EdgeDomain(rs.getString("URL_DOMAIN")),
rs.getString("COMMENT")
)
);
}
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
return ret;
}
}

37
code/services-core/control-service/src/main/java/nu/marginalia/control/svc/DomainComplaintService.java
View File

@ -18,10 +18,14 @@ import java.util.Optional;
*/
public class DomainComplaintService {
private final HikariDataSource dataSource;
private final ControlBlacklistService blacklistService;
@Inject
public DomainComplaintService(HikariDataSource dataSource) {
public DomainComplaintService(HikariDataSource dataSource,
ControlBlacklistService blacklistService
) {
this.dataSource = dataSource;
this.blacklistService = blacklistService;
}
public List<DomainComplaintModel> getComplaints() {
@ -53,12 +57,13 @@ public class DomainComplaintService {
}
public void approveAppealBlacklisting(EdgeDomain domain) {
removeFromBlacklist(domain);
blacklistService.removeFromBlacklist(domain);
setDecision(domain, "APPROVED");
}
public void blacklistDomain(EdgeDomain domain) {
addToBlacklist(domain);
blacklistService.addToBlacklist(domain, "Domain complaint");
setDecision(domain, "BLACKLISTED");
}
@ -66,33 +71,7 @@ public class DomainComplaintService {
setDecision(domain, "REJECTED");
}
private void addToBlacklist(EdgeDomain domain) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN) VALUES (?)
""")) {
stmt.setString(1, domain.toString());
stmt.executeUpdate();
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
private void removeFromBlacklist(EdgeDomain domain) {
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?
""")) {
stmt.setString(1, domain.toString());
stmt.addBatch();
stmt.setString(1, domain.domain);
stmt.executeBatch();
}
catch (SQLException ex) {
throw new RuntimeException(ex);
}
}
private void setDecision(EdgeDomain domain, String decision) {
try (var conn = dataSource.getConnection();

68
code/services-core/control-service/src/main/resources/templates/control/blacklist.hdb Normal file
View File

@ -0,0 +1,68 @@
<!DOCTYPE html>
<html>
<head>
<title>Control Service</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="/style.css" />
</head>
<body>
{{> control/partials/nav}}
<section>
<h1>Blacklist</h1>
<p>
The blacklist is a list of sanctioned domains that will not be
crawled, indexed, or returned from the search results.
</p>
<table style="max-width: 80ch">
<tr>
<th>Description</th><th>Action</th>
</tr>
<tr>
<td><b>Add To Blacklist</b><p>
This will add the given domain to the blacklist.
</td>
<td>
<form method="post" action="/blacklist?act=add" onsubmit="return confirm('Confirm addition')">
<label for="add-domain">Domain: </label>&nbsp;<input type="text" id="add-domain" name="domain" placeholder="Domain" /><br>
<label for="comment">Comment: </label>&nbsp;<input type="text" id="comment" name="comment" value="Manually Added" />
<br>
<br>
<input type="submit" value="Add">
</form>
</td>
</tr>
<tr>
<td><b>Remove from blacklist</b><p>
Remove the specified domain from the blacklist. This will ensure that
the domain is not blacklisted, in doing so it may remove the root domain
from the blacklist as well.
</td>
<td>
<form method="post" action="/blacklist?act=del" onsubmit="return confirm('Confirm removal')">
<label for="rm-domain">Domain: </label>&nbsp;<input type="text" id="rm-domain" name="domain" placeholder="Domain" />
<br>
<br>
<input type="submit" value="Add">
</form>
</td>
</tr>
</table>
<h1>Recent Additions</h1>
<table>
<tr>
<th>Domain</th>
<th>Comment</th>
</tr>
{{#each blacklist}}
<tr>
<td>{{domain}}</td>
<td>{{comment}}</td>
</tr>
{{/each}}
</table>
</section>
</body>
</html>