diff --git a/code/services-core/control-service/java/nu/marginalia/control/ControlService.java b/code/services-core/control-service/java/nu/marginalia/control/ControlService.java index 5c0a014e..8a509a88 100644 --- a/code/services-core/control-service/java/nu/marginalia/control/ControlService.java +++ b/code/services-core/control-service/java/nu/marginalia/control/ControlService.java @@ -2,16 +2,18 @@ package nu.marginalia.control; import com.google.gson.Gson; import com.google.inject.Inject; -import nu.marginalia.service.ServiceMonitors; import nu.marginalia.control.actor.ControlActorService; import nu.marginalia.control.app.svc.*; -import nu.marginalia.control.node.svc.ControlNodeActionsService; import nu.marginalia.control.node.svc.ControlFileStorageService; +import nu.marginalia.control.node.svc.ControlNodeActionsService; import nu.marginalia.control.node.svc.ControlNodeService; import nu.marginalia.control.sys.svc.*; import nu.marginalia.model.gson.GsonFactory; import nu.marginalia.screenshot.ScreenshotService; -import nu.marginalia.service.server.*; +import nu.marginalia.service.ServiceMonitors; +import nu.marginalia.service.server.BaseServiceParams; +import nu.marginalia.service.server.Service; +import nu.marginalia.service.server.StaticResources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -19,7 +21,7 @@ import spark.Response; import spark.Spark; import java.io.IOException; -import java.util.*; +import java.util.Map; public class ControlService extends Service { @@ -56,6 +58,7 @@ public class ControlService extends Service { ControlDomainRankingSetsService controlDomainRankingSetsService, ControlActorService controlActorService, AbortedProcessService abortedProcessService, + DomainsManagementService domainsManagementService, ControlErrorHandler errorHandler ) throws IOException { @@ -84,6 +87,7 @@ public class ControlService extends Service { apiKeyService.register(); domainComplaintService.register(); randomExplorationService.register(); + domainsManagementService.register(); errorHandler.register(); diff --git a/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java new file mode 100644 index 00000000..c9c09598 --- /dev/null +++ b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java @@ -0,0 +1,39 @@ +package nu.marginalia.control.app.model; + +public record DomainModel(int id, + String name, + String ip, + int nodeAffinity, + double rank, + boolean blacklisted) { + public boolean isIndexed() { + return nodeAffinity > 0; + } + + public DomainAffinityState getAffinityState() { + if (nodeAffinity < 0) { + return DomainAffinityState.Known; + } + else if (nodeAffinity == 0) { + return DomainAffinityState.Scheduled; + } + else { + return DomainAffinityState.Assigned; + } + } + + public enum DomainAffinityState { + Assigned("The domain has been assigned to a node."), + Scheduled("The domain will be assigned to the next crawling node."), + Known("The domain is known but not yet scheduled for crawling."); + + private final String desc; + DomainAffinityState(String desc) { + this.desc = desc; + } + + public String getDesc() { + return desc; + } + } +} diff --git a/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java new file mode 100644 index 00000000..f512bb43 --- /dev/null +++ b/code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java @@ -0,0 +1,25 @@ +package nu.marginalia.control.app.model; + +import java.util.List; +import java.util.Map; + +public record DomainSearchResultModel(String query, + String affinity, + String field, + Map selectedAffinity, + Map selectedField, + int page, + boolean hasNext, + boolean hasPrevious, + List results) +{ + public Integer getNextPage() { + if (!hasNext) return null; + return page + 1; + } + + public Integer getPreviousPage() { + if (!hasPrevious) return null; + return page - 1; + } +} diff --git a/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java b/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java new file mode 100644 index 00000000..f5b2b42c --- /dev/null +++ b/code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java @@ -0,0 +1,127 @@ +package nu.marginalia.control.app.svc; + +import com.google.inject.Inject; +import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.control.ControlRendererFactory; +import nu.marginalia.control.app.model.DomainModel; +import nu.marginalia.control.app.model.DomainSearchResultModel; +import spark.Request; +import spark.Response; +import spark.Spark; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class DomainsManagementService { + + private final HikariDataSource dataSource; + private final ControlRendererFactory rendererFactory; + + @Inject + public DomainsManagementService(HikariDataSource dataSource, + ControlRendererFactory rendererFactory + ) { + this.dataSource = dataSource; + this.rendererFactory = rendererFactory; + } + + public void register() throws IOException { + + var domainsViewRenderer = rendererFactory.renderer("control/app/domains"); + + Spark.get("/domains", this::getDomains, domainsViewRenderer::render); + + } + + private DomainSearchResultModel getDomains(Request request, Response response) throws SQLException { + List ret = new ArrayList<>(); + + String filterRaw = Objects.requireNonNullElse(request.queryParams("filter"), "*"); + + String filter; + if (filterRaw.isBlank()) filter = "%"; + else filter = filterRaw.replace('*', '%'); + + int page = Integer.parseInt(Objects.requireNonNullElse(request.queryParams("page"), "0")); + boolean hasMore = false; + int count = 10; + + String field = Objects.requireNonNullElse(request.queryParams("field"), "domain"); + Map selectedField = Map.of(field, true); + + String affinity = Objects.requireNonNullElse(request.queryParams("affinity"), "all"); + Map selectedAffinity = Map.of(affinity, true); + + + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(""" + SELECT EC_DOMAIN.ID, + DOMAIN_NAME, + NODE_AFFINITY, + `RANK`, + IP, + EC_DOMAIN_BLACKLIST.URL_DOMAIN IS NOT NULL AS BLACKLISTED + FROM WMSA_prod.EC_DOMAIN + LEFT JOIN WMSA_prod.EC_DOMAIN_BLACKLIST ON DOMAIN_NAME = EC_DOMAIN_BLACKLIST.URL_DOMAIN + + """ + + // listen, I wouldn't worry about it + (switch (field) { + case "domain" -> "WHERE DOMAIN_NAME LIKE ?"; + case "ip" -> "WHERE IP LIKE ?"; + case "id" -> "WHERE EC_DOMAIN.ID = ?"; + default -> "WHERE DOMAIN_NAME LIKE ?"; + }) + + " " + + (switch (affinity) { + case "assigned" -> "AND NODE_AFFINITY > 0"; + case "scheduled" -> "AND NODE_AFFINITY = 0"; + case "known" -> "AND NODE_AFFINITY < 0"; + default -> ""; + }) + + + """ + + LIMIT ? + OFFSET ? + """ + )) + { + stmt.setString(1, filter); + stmt.setInt(2, count + 1); + stmt.setInt(3, count * page); + + try (var rs = stmt.executeQuery()) { + while (rs.next()) { + if (ret.size() == count) { + hasMore = true; + break; + } + ret.add(new DomainModel( + rs.getInt("ID"), + rs.getString("DOMAIN_NAME"), + rs.getString("IP"), + rs.getInt("NODE_AFFINITY"), + Math.round(100 * rs.getDouble("RANK"))/100., + rs.getBoolean("BLACKLISTED") + )); + } + } + } + + return new DomainSearchResultModel(filterRaw, + affinity, + field, + selectedAffinity, + selectedField, + page, + hasMore, + page > 0, + ret); + } + +} diff --git a/code/services-core/control-service/resources/templates/control/app/domains.hdb b/code/services-core/control-service/resources/templates/control/app/domains.hdb new file mode 100644 index 00000000..935ee710 --- /dev/null +++ b/code/services-core/control-service/resources/templates/control/app/domains.hdb @@ -0,0 +1,74 @@ + + + + Control Service + {{> control/partials/head-includes }} + + +{{> control/partials/nav}} +
+

Domains

+ + + + + + + + + + + + + + + + + + + {{#each results}} + + + + + + + + + {{/each}} + {{#unless results}} + + + + {{/unless}} + + + + + +
+ + + +
DomainIDNode AffinityRankIPBlacklisted
{{name}}{{id}}{{affinityState}} ({{nodeAffinity}}){{rank}}{{ip}}{{#if blacklisted}}✓{{/if}}
No results found
+ {{#if hasPrevious}} + Previous + {{/if}} + + {{#if hasNext}} + Next + {{/if}} +
+
+ +{{> control/partials/foot-includes }} + \ No newline at end of file diff --git a/code/services-core/control-service/resources/templates/control/partials/nav.hdb b/code/services-core/control-service/resources/templates/control/partials/nav.hdb index 63297bd9..c6186ae1 100644 --- a/code/services-core/control-service/resources/templates/control/partials/nav.hdb +++ b/code/services-core/control-service/resources/templates/control/partials/nav.hdb @@ -23,6 +23,7 @@ {{/unless}} +
  • Domains