mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-02-23 21:18:58 +00:00
(control) New view for domains
Add capability to assign domains, and bulk-add new domains.
This commit is contained in:
Viktor Lofgren
parent
74e25370ca
commit
b1bfe6f76e
@ -6,8 +6,9 @@ public record DomainModel(int id,
|
||||
int nodeAffinity,
|
||||
double rank,
|
||||
boolean blacklisted) {
|
||||
public boolean isIndexed() {
|
||||
return nodeAffinity > 0;
|
||||
|
||||
public boolean isUnassigned() {
|
||||
return nodeAffinity < 0;
|
||||
}
|
||||
|
||||
public DomainAffinityState getAffinityState() {
|
||||
|
||||
@ -11,6 +11,7 @@ public record DomainSearchResultModel(String query,
|
||||
int page,
|
||||
boolean hasNext,
|
||||
boolean hasPrevious,
|
||||
List<Integer> nodes,
|
||||
List<DomainModel> results)
|
||||
{
|
||||
public Integer getNextPage() {
|
||||
|
||||
@ -3,40 +3,129 @@ package nu.marginalia.control.app.svc;
|
||||
import com.google.inject.Inject;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import nu.marginalia.control.ControlRendererFactory;
|
||||
import nu.marginalia.control.Redirects;
|
||||
import nu.marginalia.control.app.model.DomainModel;
|
||||
import nu.marginalia.control.app.model.DomainSearchResultModel;
|
||||
import nu.marginalia.model.EdgeDomain;
|
||||
import nu.marginalia.nodecfg.NodeConfigurationService;
|
||||
import spark.Request;
|
||||
import spark.Response;
|
||||
import spark.Spark;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.*;
|
||||
|
||||
public class DomainsManagementService {
|
||||
|
||||
private final HikariDataSource dataSource;
|
||||
private final NodeConfigurationService nodeConfigurationService;
|
||||
private final ControlRendererFactory rendererFactory;
|
||||
|
||||
@Inject
|
||||
public DomainsManagementService(HikariDataSource dataSource,
|
||||
NodeConfigurationService nodeConfigurationService,
|
||||
ControlRendererFactory rendererFactory
|
||||
) {
|
||||
this.dataSource = dataSource;
|
||||
this.nodeConfigurationService = nodeConfigurationService;
|
||||
this.rendererFactory = rendererFactory;
|
||||
}
|
||||
|
||||
public void register() throws IOException {
|
||||
|
||||
var domainsViewRenderer = rendererFactory.renderer("control/app/domains");
|
||||
var addDomainsViewRenderer = rendererFactory.renderer("control/app/domains-new");
|
||||
var addDomainsAfterReportRenderer = rendererFactory.renderer("control/app/domains-new-report");
|
||||
|
||||
Spark.get("/domains", this::getDomains, domainsViewRenderer::render);
|
||||
Spark.get("/domain", this::getDomains, domainsViewRenderer::render);
|
||||
Spark.get("/domain/new", this::addDomains, addDomainsViewRenderer::render);
|
||||
Spark.post("/domain/new", this::addDomains, addDomainsAfterReportRenderer::render);
|
||||
Spark.post("/domain/:id/assign/:node", this::assignDomain, new Redirects.HtmlRedirect("/domain"));
|
||||
|
||||
}
|
||||
|
||||
private Object addDomains(Request request, Response response) throws SQLException {
|
||||
if ("GET".equals(request.requestMethod())) {
|
||||
return "";
|
||||
}
|
||||
else if ("POST".equals(request.requestMethod())) {
|
||||
String nodeStr = request.queryParams("node");
|
||||
String domainsStr = request.queryParams("domains");
|
||||
|
||||
int node = Integer.parseInt(nodeStr);
|
||||
String[] domains = domainsStr.split("\n+");
|
||||
|
||||
List<EdgeDomain> validDomains = new ArrayList<>();
|
||||
List<String> invalidDomains = new ArrayList<>();
|
||||
|
||||
for (String domain : domains) {
|
||||
domain = domain.trim();
|
||||
if (domain.isBlank()) continue;
|
||||
if (domain.length() > 255) {
|
||||
invalidDomains.add(domain);
|
||||
continue;
|
||||
}
|
||||
if (domain.startsWith("#")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Run through the URI parser to check for bad domains
|
||||
try {
|
||||
if (domain.contains(":")) {
|
||||
domain = new URI(domain ).toURL().getHost();
|
||||
}
|
||||
else {
|
||||
domain = new URI("https://" + domain + "/").toURL().getHost();
|
||||
}
|
||||
} catch (URISyntaxException | MalformedURLException e) {
|
||||
invalidDomains.add(domain);
|
||||
continue;
|
||||
}
|
||||
|
||||
validDomains.add(new EdgeDomain(domain));
|
||||
}
|
||||
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("INSERT IGNORE INTO EC_DOMAIN (DOMAIN_NAME, DOMAIN_TOP, NODE_AFFINITY) VALUES (?, ?, ?)"))
|
||||
{
|
||||
for (var domain : validDomains) {
|
||||
stmt.setString(1, domain.toString());
|
||||
stmt.setString(2, domain.getTopDomain());
|
||||
stmt.setInt(3, node);
|
||||
stmt.addBatch();
|
||||
}
|
||||
stmt.executeBatch();
|
||||
}
|
||||
|
||||
return Map.of("validDomains", validDomains,
|
||||
"invalidDomains", invalidDomains);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private Object assignDomain(Request request, Response response) throws SQLException {
|
||||
|
||||
String idStr = request.params(":id");
|
||||
String nodeStr = request.params(":node");
|
||||
|
||||
int id = Integer.parseInt(idStr);
|
||||
int node = Integer.parseInt(nodeStr);
|
||||
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("UPDATE EC_DOMAIN SET NODE_AFFINITY = ? WHERE ID = ?"))
|
||||
{
|
||||
stmt.setInt(1, node);
|
||||
stmt.setInt(2, id);
|
||||
stmt.executeUpdate();
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private DomainSearchResultModel getDomains(Request request, Response response) throws SQLException {
|
||||
List<DomainModel> ret = new ArrayList<>();
|
||||
|
||||
@ -56,9 +145,8 @@ public class DomainsManagementService {
|
||||
String affinity = Objects.requireNonNullElse(request.queryParams("affinity"), "all");
|
||||
Map<String, Boolean> selectedAffinity = Map.of(affinity, true);
|
||||
|
||||
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement("""
|
||||
StringJoiner queryJoiner = new StringJoiner(" ");
|
||||
queryJoiner.add("""
|
||||
SELECT EC_DOMAIN.ID,
|
||||
DOMAIN_NAME,
|
||||
NODE_AFFINITY,
|
||||
@ -67,29 +155,25 @@ public class DomainsManagementService {
|
||||
EC_DOMAIN_BLACKLIST.URL_DOMAIN IS NOT NULL AS BLACKLISTED
|
||||
FROM WMSA_prod.EC_DOMAIN
|
||||
LEFT JOIN WMSA_prod.EC_DOMAIN_BLACKLIST ON DOMAIN_NAME = EC_DOMAIN_BLACKLIST.URL_DOMAIN
|
||||
""")
|
||||
.add((switch (field) {
|
||||
case "domain" -> "WHERE DOMAIN_NAME LIKE ?";
|
||||
case "ip" -> "WHERE IP LIKE ?";
|
||||
case "id" -> "WHERE EC_DOMAIN.ID = ?";
|
||||
default -> "WHERE DOMAIN_NAME LIKE ?";
|
||||
}))
|
||||
.add((switch (affinity) {
|
||||
case "assigned" -> "AND NODE_AFFINITY > 0";
|
||||
case "scheduled" -> "AND NODE_AFFINITY = 0";
|
||||
case "unassigned" -> "AND NODE_AFFINITY < 0";
|
||||
default -> "";
|
||||
}))
|
||||
.add("LIMIT ?")
|
||||
.add("OFFSET ?");
|
||||
|
||||
"""
|
||||
+ // listen, I wouldn't worry about it
|
||||
(switch (field) {
|
||||
case "domain" -> "WHERE DOMAIN_NAME LIKE ?";
|
||||
case "ip" -> "WHERE IP LIKE ?";
|
||||
case "id" -> "WHERE EC_DOMAIN.ID = ?";
|
||||
default -> "WHERE DOMAIN_NAME LIKE ?";
|
||||
})
|
||||
+ " " +
|
||||
(switch (affinity) {
|
||||
case "assigned" -> "AND NODE_AFFINITY > 0";
|
||||
case "scheduled" -> "AND NODE_AFFINITY = 0";
|
||||
case "known" -> "AND NODE_AFFINITY < 0";
|
||||
default -> "";
|
||||
}) +
|
||||
|
||||
"""
|
||||
|
||||
LIMIT ?
|
||||
OFFSET ?
|
||||
"""
|
||||
))
|
||||
try (var conn = dataSource.getConnection();
|
||||
var stmt = conn.prepareStatement(queryJoiner.toString()))
|
||||
{
|
||||
stmt.setString(1, filter);
|
||||
stmt.setInt(2, count + 1);
|
||||
@ -113,6 +197,12 @@ public class DomainsManagementService {
|
||||
}
|
||||
}
|
||||
|
||||
List<Integer> nodes = new ArrayList<>();
|
||||
|
||||
for (var node : nodeConfigurationService.getAll()) {
|
||||
nodes.add(node.node());
|
||||
}
|
||||
|
||||
return new DomainSearchResultModel(filterRaw,
|
||||
affinity,
|
||||
field,
|
||||
@ -121,6 +211,7 @@ public class DomainsManagementService {
|
||||
page,
|
||||
hasMore,
|
||||
page > 0,
|
||||
nodes,
|
||||
ret);
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en-US">
|
||||
<head>
|
||||
<title>Control Service</title>
|
||||
{{> control/partials/head-includes }}
|
||||
</head>
|
||||
<body>
|
||||
{{> control/partials/nav}}
|
||||
<div class="container">
|
||||
<h1 class="my-3">Add Domains Report</h1>
|
||||
|
||||
<p></p>
|
||||
{{#unless invalidDomains}}
|
||||
<p>All domains were added successfully!</p>
|
||||
{{/unless}}
|
||||
{{#if invalidDomains}}
|
||||
<p>Some domains were invalid and could not be added:</p>
|
||||
<textarea class="form-control" rows="10" disabled>
|
||||
{{#each invalidDomains}}{{.}}{{/each}}
|
||||
</textarea>
|
||||
{{/if}}
|
||||
<p></p>
|
||||
</div>
|
||||
</body>
|
||||
{{> control/partials/foot-includes }}
|
||||
</html>
|
||||
@ -0,0 +1,42 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en-US">
|
||||
<head>
|
||||
<title>Control Service</title>
|
||||
{{> control/partials/head-includes }}
|
||||
</head>
|
||||
<body>
|
||||
{{> control/partials/nav}}
|
||||
<div class="container">
|
||||
<h1 class="my-3">Add Domains</h1>
|
||||
|
||||
<form method="post">
|
||||
<div class="form-group my-3">
|
||||
<label for="domains" class="form-label">Domains to add</label>
|
||||
<textarea name="domains" class="form-control" rows="10"></textarea>
|
||||
<span class="text-muted">
|
||||
Enter a list of domains to add, one per line. The system will check if the domain is already in the database and
|
||||
will not add duplicates. Spaces and empty lines are ignored.
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="form-group my-3">
|
||||
<label for="node" class="form-label">Node</label>
|
||||
<select name="node" class="form-select">
|
||||
<option value="-1">Unassigned</option>
|
||||
<option value="0" selected>Auto</option>
|
||||
{{#each global-context.nodes}}
|
||||
<option value="{{this}}">Node {{id}}</option>
|
||||
{{/each}}
|
||||
|
||||
</select>
|
||||
<span class="text-muted">
|
||||
Select the node to assign the domains to, this is the index node that will "own" the domain, crawl its documents
|
||||
and index dem. If you select "Auto", the system will assign the domains to the next node that performs a crawl.
|
||||
</span>
|
||||
</div>
|
||||
<button type="submit" class="btn btn-primary">Add</button>
|
||||
</form>
|
||||
</div>
|
||||
</body>
|
||||
{{> control/partials/foot-includes }}
|
||||
</html>
|
||||
@ -23,7 +23,7 @@
|
||||
<td>
|
||||
<select name="affinity" class="form-select" aria-label="Select Node Affinity">
|
||||
<option value="all" {{#if selectedAffinity.all}}selected{{/if}}>-</option>
|
||||
<option value="known" {{#if selectedAffinity.known}}selected{{/if}}>Known</option>
|
||||
<option value="unassigned" {{#if selectedAffinity.unassigned}}selected{{/if}}>Unassigned</option>
|
||||
<option value="scheduled" {{#if selectedAffinity.scheduled}}selected{{/if}}>Scheduled</option>
|
||||
<option value="assigned" {{#if selectedAffinity.assigned}}selected{{/if}}>Assigned</option>
|
||||
</select>
|
||||
@ -34,7 +34,7 @@
|
||||
<tr>
|
||||
<th>Domain</th>
|
||||
<th>ID</th>
|
||||
<th>Node Affinity</th>
|
||||
<th title="Which, if any, index node owns a domain and will crawl and index it">Node Affinity</th>
|
||||
<th>Rank</th>
|
||||
<th>IP</th>
|
||||
<th>Blacklisted</th>
|
||||
@ -43,7 +43,42 @@
|
||||
<tr>
|
||||
<td>{{name}}</td>
|
||||
<td>{{id}}</td>
|
||||
<td title="{{affinityState.desc}}">{{affinityState}} ({{nodeAffinity}})</td>
|
||||
<td title="{{affinityState.desc}}">{{#unless unassigned}}{{affinityState}} {{#if nodeAffinity}}{{nodeAffinity}}{{/if}} {{/unless}}
|
||||
{{#if unassigned}}
|
||||
<div class="dropdown">
|
||||
<button title="Assign to a node" class="btn btn-secondary dropdown-toggle" type="button" id="dropdownMenuButton1" data-bs-toggle="dropdown" aria-expanded="false">
|
||||
Unassigned
|
||||
</button>
|
||||
<ul class="dropdown-menu" aria-labelledby="dropdownMenuButton1">
|
||||
<form method="post">
|
||||
<input type="hidden" name="node" value="0">
|
||||
<li>
|
||||
<button
|
||||
class="dropdown-item"
|
||||
title="Assign to the next node that performs a crawl"
|
||||
formaction="/domain/{{id}}/assign/0"
|
||||
type="submit">
|
||||
Any
|
||||
</button>
|
||||
</li>
|
||||
|
||||
{{#each nodes}}
|
||||
<input type="hidden" name="node" value="{{.}}">
|
||||
<li>
|
||||
<button
|
||||
class="dropdown-item"
|
||||
title="Assign to node {{.}}"
|
||||
formaction="/domain/{{id}}/assign/{{.}}"
|
||||
type="submit">
|
||||
Node {{.}}
|
||||
</button>
|
||||
</li>
|
||||
{{/each}}
|
||||
</form>
|
||||
</ul>
|
||||
</div>
|
||||
{{/if}}
|
||||
</td>
|
||||
<td>{{rank}}</td>
|
||||
<td>{{ip}}</td>
|
||||
<td>{{#if blacklisted}}✓{{/if}}</td>
|
||||
@ -60,7 +95,7 @@
|
||||
<a href="?page={{previousPage}}&filter={{query}}&field={{field}}&affinity={{affinity}}">Previous</a>
|
||||
{{/if}}
|
||||
</td>
|
||||
<td colspan="3"></td>
|
||||
<td colspan="4"></td>
|
||||
<td>
|
||||
{{#if hasNext}}
|
||||
<a href="?page={{nextPage}}&filter={{query}}&field={{field}}&affinity={{affinity}}">Next</a>
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
<script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.8/dist/umd/popper.min.js" integrity="sha384-I7E8VVD/ismYTF4hNIPjVp/Zjvgyol6VFvRkX/vR+Vc4jQkC+hVqc2pM8ODewa9r" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.min.js" integrity="sha384-BBtl+eGJRgqQAUMxJ7pMwbEyER4l1g+O15P+16Ep7Q9Q+zqX6gSbd85u4mG4QzX+" crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
|
||||
<script src="/refresh.js"></script>
|
||||
<script type="javascript">
|
||||
|
||||
@ -16,14 +16,21 @@
|
||||
<a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button" aria-expanded="false">Application</a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a class="dropdown-item" href="/api-keys" title="Create or remove API keys">API Keys</a></li>
|
||||
<li><a class="dropdown-item" href="/blacklist" title="Add or remove website sanctions">Blacklist</a></li>
|
||||
<li><a class="dropdown-item" href="/search-to-ban" title="Search function for easy blacklisting">Blacklist Search</a></li>
|
||||
<li><a class="dropdown-item" href="/complaints" title="View and act on user complaints">Complaints</a></li>
|
||||
<li><a class="dropdown-item" href="/review-random-domains" title="Review random domains list">Random Exploration</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
{{/unless}}
|
||||
<li class="nav-item"><a class="nav-link" href="/domains">Domains</a></li>
|
||||
<li class="nav-item dropdown">
|
||||
<a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button" aria-expanded="false">Domains</a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a class="dropdown-item" href="/domain/new" title="Add New Domains">Add Domains</a></li>
|
||||
<li><a class="dropdown-item" href="/domain" title="List Domains">Manage Domains</a></li>
|
||||
<li><hr class="dropdown-divider"></li>
|
||||
<li><a class="dropdown-item" href="/blacklist" title="Add or remove website sanctions">Blacklist</a></li>
|
||||
<li><a class="dropdown-item" href="/search-to-ban" title="Search function for easy blacklisting">Blacklist Search</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="nav-item dropdown">
|
||||
<a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button" aria-expanded="false">Index Nodes</a>
|
||||
<ul class="dropdown-menu">
|
||||
|
||||
Loading…
Reference in New Issue
Block a user