mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-02-23 21:18:58 +00:00
(blacklist) Blacklist management
This commit is contained in:
parent
807fb2d052
commit
4f8048be31
@ -2,6 +2,7 @@
|
||||
CREATE TABLE IF NOT EXISTS EC_DOMAIN_BLACKLIST (
|
||||
ID INT PRIMARY KEY AUTO_INCREMENT,
|
||||
URL_DOMAIN VARCHAR(255) UNIQUE NOT NULL
|
||||
COMMENT VARCHAR(255) DEFAULT NULL
|
||||
)
|
||||
CHARACTER SET utf8mb4
|
||||
COLLATE utf8mb4_unicode_ci;
|
||||
|
@ -24,6 +24,7 @@ import java.sql.SQLException;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ControlService extends Service {
|
||||
@ -36,6 +37,7 @@ public class ControlService extends Service {
|
||||
private final EventLogService eventLogService;
|
||||
private final ApiKeyService apiKeyService;
|
||||
private final DomainComplaintService domainComplaintService;
|
||||
private final ControlBlacklistService blacklistService;
|
||||
private final ControlActorService controlActorService;
|
||||
private final StaticResources staticResources;
|
||||
private final MessageQueueService messageQueueService;
|
||||
@ -54,6 +56,7 @@ public class ControlService extends Service {
|
||||
ControlFileStorageService controlFileStorageService,
|
||||
ApiKeyService apiKeyService,
|
||||
DomainComplaintService domainComplaintService,
|
||||
ControlBlacklistService blacklistService,
|
||||
ControlActionsService controlActionsService
|
||||
) throws IOException {
|
||||
|
||||
@ -63,6 +66,7 @@ public class ControlService extends Service {
|
||||
this.eventLogService = eventLogService;
|
||||
this.apiKeyService = apiKeyService;
|
||||
this.domainComplaintService = domainComplaintService;
|
||||
this.blacklistService = blacklistService;
|
||||
|
||||
var indexRenderer = rendererFactory.renderer("control/index");
|
||||
var servicesRenderer = rendererFactory.renderer("control/services");
|
||||
@ -85,6 +89,7 @@ public class ControlService extends Service {
|
||||
var viewMessageRenderer = rendererFactory.renderer("control/view-message");
|
||||
|
||||
var actionsViewRenderer = rendererFactory.renderer("control/actions");
|
||||
var blacklistRenderer = rendererFactory.renderer("control/blacklist");
|
||||
|
||||
this.controlActorService = controlActorService;
|
||||
|
||||
@ -109,6 +114,7 @@ public class ControlService extends Service {
|
||||
final HtmlRedirect redirectToActors = new HtmlRedirect("/actors");
|
||||
final HtmlRedirect redirectToApiKeys = new HtmlRedirect("/api-keys");
|
||||
final HtmlRedirect redirectToStorage = new HtmlRedirect("/storage");
|
||||
final HtmlRedirect redirectToBlacklist = new HtmlRedirect("/blacklist");
|
||||
final HtmlRedirect redirectToComplaints = new HtmlRedirect("/complaints");
|
||||
final HtmlRedirect redirectToMessageQueue = new HtmlRedirect("/message-queue");
|
||||
|
||||
@ -145,6 +151,11 @@ public class ControlService extends Service {
|
||||
Spark.post("/public/storage/specs", controlActorService::createCrawlSpecification, redirectToStorage);
|
||||
Spark.post("/public/storage/:fid/delete", controlFileStorageService::flagFileForDeletionRequest, redirectToStorage);
|
||||
|
||||
// Blacklist
|
||||
|
||||
Spark.get("/public/blacklist", this::blacklistModel, blacklistRenderer::render);
|
||||
Spark.post("/public/blacklist", this::updateBlacklist, redirectToBlacklist);
|
||||
|
||||
// API Keys
|
||||
|
||||
Spark.get("/public/api-keys", this::apiKeysModel, apiKeysRenderer::render);
|
||||
@ -171,6 +182,22 @@ public class ControlService extends Service {
|
||||
monitors.subscribe(this::logMonitorStateChange);
|
||||
}
|
||||
|
||||
private Object blacklistModel(Request request, Response response) {
|
||||
return Map.of("blacklist", blacklistService.lastNAdditions(100));
|
||||
}
|
||||
|
||||
private Object updateBlacklist(Request request, Response response) {
|
||||
var domain = new EdgeDomain(request.queryParams("domain"));
|
||||
if ("add".equals(request.queryParams("act"))) {
|
||||
var comment = Objects.requireNonNullElse(request.queryParams("comment"), "");
|
||||
blacklistService.addToBlacklist(domain, comment);
|
||||
} else if ("del".equals(request.queryParams("act"))) {
|
||||
blacklistService.removeFromBlacklist(domain);
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private Object overviewModel(Request request, Response response) {
|
||||
|
||||
return Map.of("processes", heartbeatService.getProcessHeartbeats(),
|
||||
|
@ -0,0 +1,6 @@
|
||||
package nu.marginalia.control.model;
|
||||
|
||||
import nu.marginalia.model.EdgeDomain;
|
||||
|
||||
public record BlacklistedDomainModel(EdgeDomain domain, String comment) {
|
||||
}
|
@ -0,0 +1,79 @@
|
||||
package nu.marginalia.control.svc;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import nu.marginalia.control.model.BlacklistedDomainModel;
|
||||
import nu.marginalia.model.EdgeDomain;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class ControlBlacklistService {
|
||||
|
||||
private final HikariDataSource dataSource;
|
||||
|
||||
@Inject
|
||||
public ControlBlacklistService(HikariDataSource dataSource) {
|
||||
this.dataSource = dataSource;
|
||||
}
|
||||
|
||||
public void addToBlacklist(EdgeDomain domain, String comment) {
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("""
|
||||
INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN, COMMENT) VALUES (?, ?)
|
||||
""")) {
|
||||
stmt.setString(1, domain.toString());
|
||||
stmt.setString(2, comment);
|
||||
stmt.executeUpdate();
|
||||
}
|
||||
catch (SQLException ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void removeFromBlacklist(EdgeDomain domain) {
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("""
|
||||
DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?
|
||||
""")) {
|
||||
stmt.setString(1, domain.toString());
|
||||
stmt.addBatch();
|
||||
stmt.setString(1, domain.domain);
|
||||
stmt.addBatch();
|
||||
stmt.executeBatch();
|
||||
}
|
||||
catch (SQLException ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
public List<BlacklistedDomainModel> lastNAdditions(int n) {
|
||||
final List<BlacklistedDomainModel> ret = new ArrayList<>(n);
|
||||
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("""
|
||||
SELECT URL_DOMAIN, COMMENT
|
||||
FROM EC_DOMAIN_BLACKLIST
|
||||
ORDER BY ID DESC
|
||||
LIMIT ?
|
||||
""")) {
|
||||
stmt.setInt(1, n);
|
||||
|
||||
var rs = stmt.executeQuery();
|
||||
while (rs.next()) {
|
||||
ret.add(new BlacklistedDomainModel(
|
||||
new EdgeDomain(rs.getString("URL_DOMAIN")),
|
||||
rs.getString("COMMENT")
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
catch (SQLException ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
||||
}
|
||||
}
|
@ -18,10 +18,14 @@ import java.util.Optional;
|
||||
*/
|
||||
public class DomainComplaintService {
|
||||
private final HikariDataSource dataSource;
|
||||
private final ControlBlacklistService blacklistService;
|
||||
|
||||
@Inject
|
||||
public DomainComplaintService(HikariDataSource dataSource) {
|
||||
public DomainComplaintService(HikariDataSource dataSource,
|
||||
ControlBlacklistService blacklistService
|
||||
) {
|
||||
this.dataSource = dataSource;
|
||||
this.blacklistService = blacklistService;
|
||||
}
|
||||
|
||||
public List<DomainComplaintModel> getComplaints() {
|
||||
@ -53,12 +57,13 @@ public class DomainComplaintService {
|
||||
}
|
||||
|
||||
public void approveAppealBlacklisting(EdgeDomain domain) {
|
||||
removeFromBlacklist(domain);
|
||||
blacklistService.removeFromBlacklist(domain);
|
||||
setDecision(domain, "APPROVED");
|
||||
}
|
||||
|
||||
public void blacklistDomain(EdgeDomain domain) {
|
||||
addToBlacklist(domain);
|
||||
blacklistService.addToBlacklist(domain, "Domain complaint");
|
||||
|
||||
setDecision(domain, "BLACKLISTED");
|
||||
}
|
||||
|
||||
@ -66,33 +71,7 @@ public class DomainComplaintService {
|
||||
setDecision(domain, "REJECTED");
|
||||
}
|
||||
|
||||
private void addToBlacklist(EdgeDomain domain) {
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("""
|
||||
INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN) VALUES (?)
|
||||
""")) {
|
||||
stmt.setString(1, domain.toString());
|
||||
stmt.executeUpdate();
|
||||
}
|
||||
catch (SQLException ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void removeFromBlacklist(EdgeDomain domain) {
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("""
|
||||
DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?
|
||||
""")) {
|
||||
stmt.setString(1, domain.toString());
|
||||
stmt.addBatch();
|
||||
stmt.setString(1, domain.domain);
|
||||
stmt.executeBatch();
|
||||
}
|
||||
catch (SQLException ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void setDecision(EdgeDomain domain, String decision) {
|
||||
try (var conn = dataSource.getConnection();
|
||||
|
@ -0,0 +1,68 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Control Service</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="stylesheet" href="/style.css" />
|
||||
</head>
|
||||
<body>
|
||||
{{> control/partials/nav}}
|
||||
<section>
|
||||
<h1>Blacklist</h1>
|
||||
|
||||
<p>
|
||||
The blacklist is a list of sanctioned domains that will not be
|
||||
crawled, indexed, or returned from the search results.
|
||||
</p>
|
||||
|
||||
<table style="max-width: 80ch">
|
||||
<tr>
|
||||
<th>Description</th><th>Action</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><b>Add To Blacklist</b><p>
|
||||
This will add the given domain to the blacklist.
|
||||
</td>
|
||||
<td>
|
||||
<form method="post" action="/blacklist?act=add" onsubmit="return confirm('Confirm addition')">
|
||||
<label for="add-domain">Domain: </label> <input type="text" id="add-domain" name="domain" placeholder="Domain" /><br>
|
||||
<label for="comment">Comment: </label> <input type="text" id="comment" name="comment" value="Manually Added" />
|
||||
<br>
|
||||
<br>
|
||||
<input type="submit" value="Add">
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><b>Remove from blacklist</b><p>
|
||||
Remove the specified domain from the blacklist. This will ensure that
|
||||
the domain is not blacklisted, in doing so it may remove the root domain
|
||||
from the blacklist as well.
|
||||
</td>
|
||||
<td>
|
||||
<form method="post" action="/blacklist?act=del" onsubmit="return confirm('Confirm removal')">
|
||||
<label for="rm-domain">Domain: </label> <input type="text" id="rm-domain" name="domain" placeholder="Domain" />
|
||||
<br>
|
||||
<br>
|
||||
<input type="submit" value="Add">
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<h1>Recent Additions</h1>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Domain</th>
|
||||
<th>Comment</th>
|
||||
</tr>
|
||||
{{#each blacklist}}
|
||||
<tr>
|
||||
<td>{{domain}}</td>
|
||||
<td>{{comment}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</section>
|
||||
</body>
|
||||
</html>
|
Loading…
Reference in New Issue
Block a user