Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(control) New view for domains

Browse Source 
Still a work in progress, but at this point it's possible to use for viewing domains
This commit is contained in:
Viktor Lofgren 2024-08-29 15:40:40 +02:00
parent 2f38c95886
commit 74e25370ca
6 changed files with 274 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
code/services-core/control-service/java/nu/marginalia/control/ControlService.java
View File

@ -2,16 +2,18 @@ package nu.marginalia.control;
import com.google.gson.Gson;
import com.google.inject.Inject;
import nu.marginalia.service.ServiceMonitors;
import nu.marginalia.control.actor.ControlActorService;
import nu.marginalia.control.app.svc.*;
import nu.marginalia.control.node.svc.ControlNodeActionsService;
import nu.marginalia.control.node.svc.ControlFileStorageService;
import nu.marginalia.control.node.svc.ControlNodeActionsService;
import nu.marginalia.control.node.svc.ControlNodeService;
import nu.marginalia.control.sys.svc.*;
import nu.marginalia.model.gson.GsonFactory;
import nu.marginalia.screenshot.ScreenshotService;
import nu.marginalia.service.server.*;
import nu.marginalia.service.ServiceMonitors;
import nu.marginalia.service.server.BaseServiceParams;
import nu.marginalia.service.server.Service;
import nu.marginalia.service.server.StaticResources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.Request;
@ -19,7 +21,7 @@ import spark.Response;
import spark.Spark;
import java.io.IOException;
import java.util.*;
import java.util.Map;
public class ControlService extends Service {
@ -56,6 +58,7 @@ public class ControlService extends Service {
ControlDomainRankingSetsService controlDomainRankingSetsService,
ControlActorService controlActorService,
AbortedProcessService abortedProcessService,
DomainsManagementService domainsManagementService,
ControlErrorHandler errorHandler
) throws IOException {
@ -84,6 +87,7 @@ public class ControlService extends Service {
apiKeyService.register();
domainComplaintService.register();
randomExplorationService.register();
domainsManagementService.register();
errorHandler.register();

39
code/services-core/control-service/java/nu/marginalia/control/app/model/DomainModel.java Normal file
View File

@ -0,0 +1,39 @@
package nu.marginalia.control.app.model;
public record DomainModel(int id,
String name,
String ip,
int nodeAffinity,
double rank,
boolean blacklisted) {
public boolean isIndexed() {
return nodeAffinity > 0;
}
public DomainAffinityState getAffinityState() {
if (nodeAffinity < 0) {
return DomainAffinityState.Known;
}
else if (nodeAffinity == 0) {
return DomainAffinityState.Scheduled;
}
else {
return DomainAffinityState.Assigned;
}
}
public enum DomainAffinityState {
Assigned("The domain has been assigned to a node."),
Scheduled("The domain will be assigned to the next crawling node."),
Known("The domain is known but not yet scheduled for crawling.");
private final String desc;
DomainAffinityState(String desc) {
this.desc = desc;
}
public String getDesc() {
return desc;
}
}
}

25
code/services-core/control-service/java/nu/marginalia/control/app/model/DomainSearchResultModel.java Normal file
View File

@ -0,0 +1,25 @@
package nu.marginalia.control.app.model;
import java.util.List;
import java.util.Map;
public record DomainSearchResultModel(String query,
String affinity,
String field,
Map<String, Boolean> selectedAffinity,
Map<String, Boolean> selectedField,
int page,
boolean hasNext,
boolean hasPrevious,
List<DomainModel> results)
{
public Integer getNextPage() {
if (!hasNext) return null;
return page + 1;
}
public Integer getPreviousPage() {
if (!hasPrevious) return null;
return page - 1;
}
}

127
code/services-core/control-service/java/nu/marginalia/control/app/svc/DomainsManagementService.java Normal file
View File

@ -0,0 +1,127 @@
package nu.marginalia.control.app.svc;
import com.google.inject.Inject;
import com.zaxxer.hikari.HikariDataSource;
import nu.marginalia.control.ControlRendererFactory;
import nu.marginalia.control.app.model.DomainModel;
import nu.marginalia.control.app.model.DomainSearchResultModel;
import spark.Request;
import spark.Response;
import spark.Spark;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public class DomainsManagementService {
private final HikariDataSource dataSource;
private final ControlRendererFactory rendererFactory;
@Inject
public DomainsManagementService(HikariDataSource dataSource,
ControlRendererFactory rendererFactory
) {
this.dataSource = dataSource;
this.rendererFactory = rendererFactory;
}
public void register() throws IOException {
var domainsViewRenderer = rendererFactory.renderer("control/app/domains");
Spark.get("/domains", this::getDomains, domainsViewRenderer::render);
}
private DomainSearchResultModel getDomains(Request request, Response response) throws SQLException {
List<DomainModel> ret = new ArrayList<>();
String filterRaw = Objects.requireNonNullElse(request.queryParams("filter"), "*");
String filter;
if (filterRaw.isBlank()) filter = "%";
else filter = filterRaw.replace('*', '%');
int page = Integer.parseInt(Objects.requireNonNullElse(request.queryParams("page"), "0"));
boolean hasMore = false;
int count = 10;
String field = Objects.requireNonNullElse(request.queryParams("field"), "domain");
Map<String, Boolean> selectedField = Map.of(field, true);
String affinity = Objects.requireNonNullElse(request.queryParams("affinity"), "all");
Map<String, Boolean> selectedAffinity = Map.of(affinity, true);
try (var conn = dataSource.getConnection();
var stmt = conn.prepareStatement("""
SELECT EC_DOMAIN.ID,
DOMAIN_NAME,
NODE_AFFINITY,
`RANK`,
IP,
EC_DOMAIN_BLACKLIST.URL_DOMAIN IS NOT NULL AS BLACKLISTED
FROM WMSA_prod.EC_DOMAIN
LEFT JOIN WMSA_prod.EC_DOMAIN_BLACKLIST ON DOMAIN_NAME = EC_DOMAIN_BLACKLIST.URL_DOMAIN
"""
+ // listen, I wouldn't worry about it
(switch (field) {
case "domain" -> "WHERE DOMAIN_NAME LIKE ?";
case "ip" -> "WHERE IP LIKE ?";
case "id" -> "WHERE EC_DOMAIN.ID = ?";
default -> "WHERE DOMAIN_NAME LIKE ?";
})
+ " " +
(switch (affinity) {
case "assigned" -> "AND NODE_AFFINITY > 0";
case "scheduled" -> "AND NODE_AFFINITY = 0";
case "known" -> "AND NODE_AFFINITY < 0";
default -> "";
}) +
"""
LIMIT ?
OFFSET ?
"""
))
{
stmt.setString(1, filter);
stmt.setInt(2, count + 1);
stmt.setInt(3, count * page);
try (var rs = stmt.executeQuery()) {
while (rs.next()) {
if (ret.size() == count) {
hasMore = true;
break;
}
ret.add(new DomainModel(
rs.getInt("ID"),
rs.getString("DOMAIN_NAME"),
rs.getString("IP"),
rs.getInt("NODE_AFFINITY"),
Math.round(100 * rs.getDouble("RANK"))/100.,
rs.getBoolean("BLACKLISTED")
));
}
}
}
return new DomainSearchResultModel(filterRaw,
affinity,
field,
selectedAffinity,
selectedField,
page,
hasMore,
page > 0,
ret);
}
}

74
code/services-core/control-service/resources/templates/control/app/domains.hdb Normal file
View File

@ -0,0 +1,74 @@
<!DOCTYPE html>
<html lang="en-US">
<head>
<title>Control Service</title>
{{> control/partials/head-includes }}
</head>
<body>
{{> control/partials/nav}}
<div class="container">
<h1 class="my-3">Domains</h1>
<table class="table">
<form method="get">
<tr>
<td>
<select name="field" class="form-select" aria-label="Select Field">
<option value="domain" {{#if selectedField.domain}}selected{{/if}}>Domain Name</option>
<option value="id" {{#if selectedField.id}}selected{{/if}}>Domain ID</option>
<option value="ip" {{#if selectedField.ip}}selected{{/if}}>IP</option>
</select>
</td>
<td colspan="3"><input type="text" name="filter" class="form-control" placeholder="Domain" value="{{query}}"></td>
<td>
<select name="affinity" class="form-select" aria-label="Select Node Affinity">
<option value="all" {{#if selectedAffinity.all}}selected{{/if}}>-</option>
<option value="known" {{#if selectedAffinity.known}}selected{{/if}}>Known</option>
<option value="scheduled" {{#if selectedAffinity.scheduled}}selected{{/if}}>Scheduled</option>
<option value="assigned" {{#if selectedAffinity.assigned}}selected{{/if}}>Assigned</option>
</select>
</td>
<td><button type="submit" class="btn btn-primary">Search</button></td>
</tr>
</form>
<tr>
<th>Domain</th>
<th>ID</th>
<th>Node Affinity</th>
<th>Rank</th>
<th>IP</th>
<th>Blacklisted</th>
</tr>
{{#each results}}
<tr>
<td>{{name}}</td>
<td>{{id}}</td>
<td title="{{affinityState.desc}}">{{affinityState}} ({{nodeAffinity}})</td>
<td>{{rank}}</td>
<td>{{ip}}</td>
<td>{{#if blacklisted}}&check;{{/if}}</td>
</tr>
{{/each}}
{{#unless results}}
<tr>
<td colspan="5">No results found</td>
</tr>
{{/unless}}
<tr>
<td>
{{#if hasPrevious}}
<a href="?page={{previousPage}}&filter={{query}}&field={{field}}&affinity={{affinity}}">Previous</a>
{{/if}}
</td>
<td colspan="3"></td>
<td>
{{#if hasNext}}
<a href="?page={{nextPage}}&filter={{query}}&field={{field}}&affinity={{affinity}}">Next</a>
{{/if}}
</td>
</tr>
</table>
</div>
</body>
{{> control/partials/foot-includes }}
</html>

1
code/services-core/control-service/resources/templates/control/partials/nav.hdb
View File

@ -23,6 +23,7 @@
</ul>
</li>
{{/unless}}
<li class="nav-item"><a class="nav-link" href="/domains">Domains</a></li>
<li class="nav-item dropdown">
<a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button" aria-expanded="false">Index Nodes</a>
<ul class="dropdown-menu">