diff --git a/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java index c9c09598..4c7b9754 100644 --- a/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java +++ b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java @@ -6,8 +6,9 @@ public record DomainModel(int id, int nodeAffinity, double rank, boolean blacklisted) { - public boolean isIndexed() { - return nodeAffinity > 0; + + public boolean isUnassigned() { + return nodeAffinity < 0; } public DomainAffinityState getAffinityState() { diff --git a/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java index f512bb43..13f69466 100644 --- a/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java +++ b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java @@ -11,6 +11,7 @@ public record DomainSearchResultModel(String query, int page, boolean hasNext, boolean hasPrevious, + List nodes, List results) { public Integer getNextPage() { diff --git a/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java index f5b2b42c..1e1fbe6d 100644 --- a/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java +++ b/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java @@ -3,40 +3,129 @@ package nu.marginalia.control.app.svc; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; import nu.marginalia.control.ControlRendererFactory; +import nu.marginalia.control.Redirects; import nu.marginalia.control.app.model.DomainModel; import nu.marginalia.control.app.model.DomainSearchResultModel; +import nu.marginalia.model.EdgeDomain; +import nu.marginalia.nodecfg.NodeConfigurationService; import spark.Request; import spark.Response; import spark.Spark; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; public class DomainsManagementService { private final HikariDataSource dataSource; + private final NodeConfigurationService nodeConfigurationService; private final ControlRendererFactory rendererFactory; @Inject public DomainsManagementService(HikariDataSource dataSource, + NodeConfigurationService nodeConfigurationService, ControlRendererFactory rendererFactory ) { this.dataSource = dataSource; + this.nodeConfigurationService = nodeConfigurationService; this.rendererFactory = rendererFactory; } public void register() throws IOException { var domainsViewRenderer = rendererFactory.renderer("control/app/domains"); + var addDomainsViewRenderer = rendererFactory.renderer("control/app/domains-new"); + var addDomainsAfterReportRenderer = rendererFactory.renderer("control/app/domains-new-report"); - Spark.get("/domains", this::getDomains, domainsViewRenderer::render); + Spark.get("/domain", this::getDomains, domainsViewRenderer::render); + Spark.get("/domain/new", this::addDomains, addDomainsViewRenderer::render); + Spark.post("/domain/new", this::addDomains, addDomainsAfterReportRenderer::render); + Spark.post("/domain/:id/assign/:node", this::assignDomain, new Redirects.HtmlRedirect("/domain")); } + private Object addDomains(Request request, Response response) throws SQLException { + if ("GET".equals(request.requestMethod())) { + return ""; + } + else if ("POST".equals(request.requestMethod())) { + String nodeStr = request.queryParams("node"); + String domainsStr = request.queryParams("domains"); + + int node = Integer.parseInt(nodeStr); + String[] domains = domainsStr.split("\n+"); + + List validDomains = new ArrayList<>(); + List invalidDomains = new ArrayList<>(); + + for (String domain : domains) { + domain = domain.trim(); + if (domain.isBlank()) continue; + if (domain.length() > 255) { + invalidDomains.add(domain); + continue; + } + if (domain.startsWith("#")) { + continue; + } + + // Run through the URI parser to check for bad domains + try { + if (domain.contains(":")) { + domain = new URI(domain ).toURL().getHost(); + } + else { + domain = new URI("https://" + domain + "/").toURL().getHost(); + } + } catch (URISyntaxException | MalformedURLException e) { + invalidDomains.add(domain); + continue; + } + + validDomains.add(new EdgeDomain(domain)); + } + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement("INSERT IGNORE INTO EC_DOMAIN (DOMAIN_NAME, DOMAIN_TOP, NODE_AFFINITY) VALUES (?, ?, ?)")) + { + for (var domain : validDomains) { + stmt.setString(1, domain.toString()); + stmt.setString(2, domain.getTopDomain()); + stmt.setInt(3, node); + stmt.addBatch(); + } + stmt.executeBatch(); + } + + return Map.of("validDomains", validDomains, + "invalidDomains", invalidDomains); + } + return ""; + } + + private Object assignDomain(Request request, Response response) throws SQLException { + + String idStr = request.params(":id"); + String nodeStr = request.params(":node"); + + int id = Integer.parseInt(idStr); + int node = Integer.parseInt(nodeStr); + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement("UPDATE EC_DOMAIN SET NODE_AFFINITY = ? WHERE ID = ?")) + { + stmt.setInt(1, node); + stmt.setInt(2, id); + stmt.executeUpdate(); + } + + return ""; + } + private DomainSearchResultModel getDomains(Request request, Response response) throws SQLException { List ret = new ArrayList<>(); @@ -56,9 +145,8 @@ public class DomainsManagementService { String affinity = Objects.requireNonNullElse(request.queryParams("affinity"), "all"); Map selectedAffinity = Map.of(affinity, true); - - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement(""" + StringJoiner queryJoiner = new StringJoiner(" "); + queryJoiner.add(""" SELECT EC_DOMAIN.ID, DOMAIN_NAME, NODE_AFFINITY, @@ -67,29 +155,25 @@ public class DomainsManagementService { EC_DOMAIN_BLACKLIST.URL_DOMAIN IS NOT NULL AS BLACKLISTED FROM WMSA_prod.EC_DOMAIN LEFT JOIN WMSA_prod.EC_DOMAIN_BLACKLIST ON DOMAIN_NAME = EC_DOMAIN_BLACKLIST.URL_DOMAIN + """) + .add((switch (field) { + case "domain" -> "WHERE DOMAIN_NAME LIKE ?"; + case "ip" -> "WHERE IP LIKE ?"; + case "id" -> "WHERE EC_DOMAIN.ID = ?"; + default -> "WHERE DOMAIN_NAME LIKE ?"; + })) + .add((switch (affinity) { + case "assigned" -> "AND NODE_AFFINITY > 0"; + case "scheduled" -> "AND NODE_AFFINITY = 0"; + case "unassigned" -> "AND NODE_AFFINITY < 0"; + default -> ""; + })) + .add("LIMIT ?") + .add("OFFSET ?"); - """ - + // listen, I wouldn't worry about it - (switch (field) { - case "domain" -> "WHERE DOMAIN_NAME LIKE ?"; - case "ip" -> "WHERE IP LIKE ?"; - case "id" -> "WHERE EC_DOMAIN.ID = ?"; - default -> "WHERE DOMAIN_NAME LIKE ?"; - }) - + " " + - (switch (affinity) { - case "assigned" -> "AND NODE_AFFINITY > 0"; - case "scheduled" -> "AND NODE_AFFINITY = 0"; - case "known" -> "AND NODE_AFFINITY < 0"; - default -> ""; - }) + - """ - - LIMIT ? - OFFSET ? - """ - )) + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(queryJoiner.toString())) { stmt.setString(1, filter); stmt.setInt(2, count + 1); @@ -113,6 +197,12 @@ public class DomainsManagementService { } } + List nodes = new ArrayList<>(); + + for (var node : nodeConfigurationService.getAll()) { + nodes.add(node.node()); + } + return new DomainSearchResultModel(filterRaw, affinity, field, @@ -121,6 +211,7 @@ public class DomainsManagementService { page, hasMore, page > 0, + nodes, ret); } diff --git a/code/services-core/control-service/resources/templates/control/app/domains-new-report.hdb b/code/services-core/control-service/resources/templates/control/app/domains-new-report.hdb new file mode 100644 index 00000000..52430882 --- /dev/null +++ b/code/services-core/control-service/resources/templates/control/app/domains-new-report.hdb @@ -0,0 +1,26 @@ + + + + Control Service + {{> control/partials/head-includes }} + + +{{> control/partials/nav}} +
+

Add Domains Report

+ +

+ {{#unless invalidDomains}} +

All domains were added successfully!

+ {{/unless}} + {{#if invalidDomains}} +

Some domains were invalid and could not be added:

+ + {{/if}} +

+
+ +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/resources/templates/control/app/domains-new.hdb b/code/services-core/control-service/resources/templates/control/app/domains-new.hdb new file mode 100644 index 00000000..257b7ea7 --- /dev/null +++ b/code/services-core/control-service/resources/templates/control/app/domains-new.hdb @@ -0,0 +1,42 @@ + + + + Control Service + {{> control/partials/head-includes }} + + +{{> control/partials/nav}} +
+

Add Domains

+ +
+
+ + + + Enter a list of domains to add, one per line. The system will check if the domain is already in the database and + will not add duplicates. Spaces and empty lines are ignored. + +
+ +
+ + + + Select the node to assign the domains to, this is the index node that will "own" the domain, crawl its documents + and index dem. If you select "Auto", the system will assign the domains to the next node that performs a crawl. + +
+ +
+
+ +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/resources/templates/control/app/domains.hdb b/code/services-core/control-service/resources/templates/control/app/domains.hdb index 935ee710..339b9664 100644 --- a/code/services-core/control-service/resources/templates/control/app/domains.hdb +++ b/code/services-core/control-service/resources/templates/control/app/domains.hdb @@ -23,7 +23,7 @@ @@ -34,7 +34,7 @@ Domain ID - Node Affinity + Node Affinity Rank IP Blacklisted @@ -43,7 +43,42 @@ {{name}} {{id}} - {{affinityState}} ({{nodeAffinity}}) + {{#unless unassigned}}{{affinityState}} {{#if nodeAffinity}}{{nodeAffinity}}{{/if}} {{/unless}} + {{#if unassigned}} +
+ +
    +
    + +
  • + +
    • + + {{#each nodes}} + +
  • + +
    • + {{/each}} +
    +
+
+ {{/if}} + {{rank}} {{ip}} {{#if blacklisted}}✓{{/if}} @@ -60,7 +95,7 @@ Previous {{/if}} - + {{#if hasNext}} Next diff --git a/code/services-core/control-service/resources/templates/control/partials/foot-includes.hdb b/code/services-core/control-service/resources/templates/control/partials/foot-includes.hdb index 1cb72fcd..3cfadd75 100644 --- a/code/services-core/control-service/resources/templates/control/partials/foot-includes.hdb +++ b/code/services-core/control-service/resources/templates/control/partials/foot-includes.hdb @@ -1,5 +1,4 @@ - - +