(refactor) Remove api:search-api

Application services should not have an API, but purely act as clients
to the core services (which should always have an API).

code/api/query-api/src/main/java/nu/marginalia/query/model/QueryParams.java

@@ -23,4 +23,19 @@ public record QueryParams(
         SearchSetIdentifier identifier
 )
 {
+    public QueryParams(String query, QueryLimits limits, SearchSetIdentifier identifier) {
+        this(query, null,
+                List.of(),
+                List.of(),
+                List.of(),
+                List.of(),
+                SpecificationLimit.none(),
+                SpecificationLimit.none(),
+                SpecificationLimit.none(),
+                SpecificationLimit.none(),
+                List.of(),
+                limits,
+                identifier
+                );
+    }
 }

code/api/search-api/build.gradle

@@ -1,33 +0,0 @@
-plugins {
-    id 'java'
-
-
-    id 'jvm-test-suite'
-}
-
-java {
-    toolchain {
-        languageVersion.set(JavaLanguageVersion.of(21))
-    }
-}
-
-dependencies {
-    implementation project(':code:common:model')
-    implementation project(':code:common:config')
-    implementation project(':code:libraries:message-queue')
-    implementation project(':code:common:service-discovery')
-    implementation project(':code:common:service-client')
-
-    implementation libs.bundles.slf4j
-
-    implementation libs.prometheus
-    implementation libs.notnull
-    implementation libs.guice
-    implementation libs.rxjava
-    implementation libs.gson
-
-    testImplementation libs.bundles.slf4j.test
-    testImplementation libs.bundles.junit
-    testImplementation libs.mockito
-
-}

code/api/search-api/readme.md

@@ -1,8 +0,0 @@
-# Search API
-
-Client and models for talking to the [search-service](../../services-core/search-service),
-implemented with the base client from  [service-client](../../common/service-client).
-
-## Central Classes
-
-* [SearchClient](src/main/java/nu/marginalia/search/client/SearchClient.java)

code/api/search-api/src/main/java/nu/marginalia/search/client/SearchClient.java

@@ -1,52 +0,0 @@
-package nu.marginalia.search.client;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import io.reactivex.rxjava3.core.Observable;
-import nu.marginalia.client.AbstractDynamicClient;
-import nu.marginalia.model.gson.GsonFactory;
-import nu.marginalia.mq.MessageQueueFactory;
-import nu.marginalia.mq.outbox.MqOutbox;
-import nu.marginalia.search.client.model.ApiSearchResults;
-import nu.marginalia.service.descriptor.ServiceDescriptors;
-import nu.marginalia.service.id.ServiceId;
-import nu.marginalia.WmsaHome;
-import nu.marginalia.client.Context;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.CheckReturnValue;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.util.UUID;
-
-@Singleton
-public class SearchClient extends AbstractDynamicClient {
-    private final Logger logger = LoggerFactory.getLogger(getClass());
-
-    private final MqOutbox outbox;
-
-    @Inject
-    public SearchClient(ServiceDescriptors descriptors,
-                        MessageQueueFactory messageQueueFactory) {
-
-        super(descriptors.forId(ServiceId.Search), WmsaHome.getHostsFile(), GsonFactory::get);
-
-        String inboxName = ServiceId.Search.name + ":" + "0";
-        String outboxName = System.getProperty("service-name", UUID.randomUUID().toString());
-
-        outbox = messageQueueFactory.createOutbox(inboxName, outboxName, UUID.randomUUID());
-
-    }
-
-
-    public MqOutbox outbox() {
-        return outbox;
-    }
-
-    @CheckReturnValue
-    public Observable<ApiSearchResults> query(Context ctx, String queryString, int count, int profile) {
-        return this.get(ctx, String.format("/api/search?query=%s&count=%d&index=%d", URLEncoder.encode(queryString, StandardCharsets.UTF_8), count, profile), ApiSearchResults.class);
-    }
-
-}

code/api/search-api/src/main/java/nu/marginalia/search/client/SearchMqEndpoints.java

@@ -1,6 +0,0 @@
-package nu.marginalia.search.client;
-
-public class SearchMqEndpoints {
-    /** Flushes the URL caches, run if significant changes have occurred in the URLs database */
-    public static final String FLUSH_CACHES = "FLUSH_CACHES";
-}

code/services-application/api-service/build.gradle

@@ -28,7 +28,9 @@ dependencies {
     implementation project(':code:common:config')
     implementation project(':code:common:service-discovery')
     implementation project(':code:common:service-client')
-    implementation project(':code:api:search-api')
+    implementation project(':code:api:query-api')
+    implementation project(':code:api:index-api')
+    implementation project(':code:features-index:index-query')
 
     implementation libs.bundles.slf4j
 

code/services-application/api-service/src/main/java/nu/marginalia/api/ApiSearchOperator.java

@@ -0,0 +1,111 @@
+package nu.marginalia.api;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import nu.marginalia.api.model.ApiSearchResult;
+import nu.marginalia.api.model.ApiSearchResultQueryDetails;
+import nu.marginalia.api.model.ApiSearchResults;
+import nu.marginalia.client.Context;
+import nu.marginalia.index.client.model.query.SearchSetIdentifier;
+import nu.marginalia.index.client.model.results.DecoratedSearchResultItem;
+import nu.marginalia.index.client.model.results.SearchResultKeywordScore;
+import nu.marginalia.index.query.limit.QueryLimits;
+import nu.marginalia.index.query.limit.SpecificationLimit;
+import nu.marginalia.index.searchset.SearchSet;
+import nu.marginalia.model.idx.WordMetadata;
+import nu.marginalia.query.client.QueryClient;
+import nu.marginalia.query.model.QueryParams;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Singleton
+public class ApiSearchOperator {
+    private final QueryClient queryClient;
+
+    @Inject
+    public ApiSearchOperator(QueryClient queryClient) {
+        this.queryClient = queryClient;
+    }
+
+    public ApiSearchResults query(Context context,
+                                  String query,
+                                  int count,
+                                  int index)
+    {
+        var rsp = queryClient.search(context, createParams(query, count, index));
+
+        return new ApiSearchResults("RESTRICTED", query,
+                rsp.results()
+                .stream()
+                .map(this::convert)
+                .sorted(Comparator.comparing(ApiSearchResult::getQuality).reversed())
+                .limit(count)
+                .collect(Collectors.toList()));
+    }
+
+    private QueryParams createParams(String query, int count, int index) {
+        SearchSetIdentifier searchSet = selectSearchSet(index);
+
+        return new QueryParams(
+                query,
+                new QueryLimits(
+                        2,
+                        Math.min(100, count),
+                        150,
+                        8192),
+                searchSet);
+    }
+
+    private SearchSetIdentifier selectSearchSet(int index) {
+        return switch (index) {
+            case 0 -> SearchSetIdentifier.NONE;
+            case 1 -> SearchSetIdentifier.SMALLWEB;
+            case 2 -> SearchSetIdentifier.RETRO;
+            case 3 -> SearchSetIdentifier.NONE;
+            case 5 -> SearchSetIdentifier.NONE;
+            default -> SearchSetIdentifier.NONE;
+        };
+    }
+
+
+
+    ApiSearchResult convert(DecoratedSearchResultItem url) {
+        List<List<ApiSearchResultQueryDetails>> details = new ArrayList<>();
+        if (url.rawIndexResult != null) {
+            var bySet = url.rawIndexResult.keywordScores.stream().collect(Collectors.groupingBy(SearchResultKeywordScore::subquery));
+
+            outer:
+            for (var entries : bySet.values()) {
+                List<ApiSearchResultQueryDetails> lst = new ArrayList<>();
+                for (var entry : entries) {
+                    var metadata = new WordMetadata(entry.encodedWordMetadata());
+                    if (metadata.isEmpty())
+                        continue outer;
+
+                    Set<String> flags = metadata.flagSet().stream().map(Object::toString).collect(Collectors.toSet());
+                    lst.add(new ApiSearchResultQueryDetails(entry.keyword, Long.bitCount(metadata.positions()), flags));
+                }
+                details.add(lst);
+            }
+        }
+
+        return new ApiSearchResult(
+                url.url.toString(),
+                url.getTitle(),
+                url.getDescription(),
+                sanitizeNaN(url.rankingScore, -100),
+                details
+        );
+    }
+
+    private double sanitizeNaN(double value, double alternative) {
+        if (!Double.isFinite(value)) {
+            return alternative;
+        }
+        return value;
+    }
+}

code/services-application/api-service/src/main/java/nu/marginalia/api/ApiService.java

@@ -3,13 +3,13 @@ package nu.marginalia.api;
 import com.google.gson.Gson;
 import com.google.inject.Inject;
 import nu.marginalia.api.model.ApiLicense;
+import nu.marginalia.api.model.ApiSearchResults;
 import nu.marginalia.api.svc.LicenseService;
 import nu.marginalia.api.svc.RateLimiterService;
 import nu.marginalia.api.svc.ResponseCache;
 import nu.marginalia.client.Context;
 import nu.marginalia.model.gson.GsonFactory;
-import nu.marginalia.search.client.SearchClient;
+import nu.marginalia.query.client.QueryClient;
-import nu.marginalia.search.client.model.ApiSearchResults;
 import nu.marginalia.service.server.*;
 import nu.marginalia.service.server.mq.MqNotification;
 import org.slf4j.Logger;
@@ -24,29 +24,32 @@ public class ApiService extends Service {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
     private final Gson gson = GsonFactory.get();
-    private final SearchClient searchClient;
+    private final QueryClient queryClient;
 
     private final ResponseCache responseCache;
     private final LicenseService licenseService;
     private final RateLimiterService rateLimiterService;
+    private final ApiSearchOperator searchOperator;
 
     // Marker for filtering out sensitive content from the persistent logs
     private final Marker queryMarker = MarkerFactory.getMarker("QUERY");
 
     @Inject
     public ApiService(BaseServiceParams params,
-                      SearchClient searchClient,
+                      QueryClient queryClient,
                       ResponseCache responseCache,
                       LicenseService licenseService,
-                      RateLimiterService rateLimiterService
+                      RateLimiterService rateLimiterService,
+                      ApiSearchOperator searchOperator
                       ) {
 
         super(params);
 
-        this.searchClient = searchClient;
+        this.queryClient = queryClient;
         this.responseCache = responseCache;
         this.licenseService = licenseService;
         this.rateLimiterService = rateLimiterService;
+        this.searchOperator = searchOperator;
 
         Spark.get("/public/api/", (rq, rsp) -> {
             rsp.redirect("https://memex.marginalia.nu/projects/edge/api.gmi");
@@ -102,8 +105,9 @@ public class ApiService extends Service {
 
         logger.info(queryMarker, "{} Search {}", license.key, query);
 
-        return searchClient.query(Context.fromRequest(request), query, count, index)
+        return searchOperator
-                .blockingFirst().withLicense(license.getLicense());
+                .query(Context.fromRequest(request), query, count, index)
+                .withLicense(license.getLicense());
     }
 
     private int intParam(Request request, String name, int defaultValue) {

code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResult.java

@@ -1,4 +1,4 @@
-package nu.marginalia.search.client.model;
+package nu.marginalia.api.model;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;

code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResultQueryDetails.java

@@ -1,4 +1,4 @@
-package nu.marginalia.search.client.model;
+package nu.marginalia.api.model;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;

code/services-application/api-service/src/main/java/nu/marginalia/api/model/ApiSearchResults.java

@@ -1,4 +1,4 @@
-package nu.marginalia.search.client.model;
+package nu.marginalia.api.model;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;

code/services-application/api-service/src/main/java/nu/marginalia/api/svc/ResponseCache.java

@@ -3,20 +3,12 @@ package nu.marginalia.api.svc;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.inject.Singleton;
-import nu.marginalia.api.model.ApiLicense;
+import nu.marginalia.api.model.*;
-import nu.marginalia.search.client.model.ApiSearchResults;
 
 import java.time.Duration;
 import java.util.Optional;
 
-/** This response cache exists entirely to help SearXNG with its rate limiting.
+/** This response cache exists entirely to help clients with its rate limiting.
- * For some reason they're hitting the API with like 5-12 identical requests.
- * <p/>
- * I've submitted an issue, they were like nah mang it works fine must
- * be something else  ¯\_(ツ)_/¯.
- * <p/>
- * So we're going to cache the API responses for a short while to mitigate the
- * impact of such shotgun queries on the ratelimit.
  */
 @Singleton
 public class ResponseCache {

code/services-application/api-service/src/test/java/nu/marginalia/api/svc/ResponseCacheTest.java

@@ -1,7 +1,6 @@
 package nu.marginalia.api.svc;
 
-import nu.marginalia.api.model.ApiLicense;
+import nu.marginalia.api.model.*;
-import nu.marginalia.search.client.model.ApiSearchResults;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;

code/services-application/search-service/build.gradle

@@ -35,7 +35,6 @@ dependencies {
     implementation project(':code:api:assistant-api')
     implementation project(':code:api:query-api')
     implementation project(':code:api:index-api')
-    implementation project(':code:api:search-api')
     implementation project(':code:common:service-discovery')
     implementation project(':code:common:service-client')
     implementation project(':code:common:renderer')

code/services-application/search-service/src/main/java/nu/marginalia/search/SearchOperator.java

@@ -61,18 +61,6 @@ public class SearchOperator {
         this.searchUnitConversionService = searchUnitConversionService;
     }
 
-    public List<UrlDetails> doApiSearch(Context ctx,
-                                        UserSearchParameters params) {
-
-        // TODO: This shouldn't route through search-service!
-        var queryParams = paramFactory.forRegularSearch(params);
-        var queryResponse = queryClient.search(ctx, queryParams);
-
-        logger.info(queryMarker, "Human terms (API): {}", Strings.join(queryResponse.searchTermsHuman(), ','));
-
-        return searchQueryService.getResultsFromQuery(queryResponse);
-    }
-
     public List<UrlDetails> doSiteSearch(Context ctx,
                                         String domain) {
 

code/services-application/search-service/src/main/java/nu/marginalia/search/SearchService.java

@@ -24,10 +24,8 @@ public class SearchService extends Service {
 
     private final WebsiteUrl websiteUrl;
     private final StaticResources staticResources;
-    private final FileStorageService fileStorageService;
 
     private static final Logger logger = LoggerFactory.getLogger(SearchService.class);
-    private final ServiceEventLog eventLog;
 
     @SneakyThrows
     @Inject
@@ -38,16 +36,12 @@ public class SearchService extends Service {
                          SearchErrorPageService errorPageService,
                          SearchAddToCrawlQueueService addToCrawlQueueService,
                          SearchFlagSiteService flagSiteService,
-                         SearchQueryService searchQueryService,
+                         SearchQueryService searchQueryService
-                         SearchApiQueryService apiQueryService,
-                         FileStorageService fileStorageService
                              ) {
         super(params);
 
-        this.eventLog = params.eventLog;
         this.websiteUrl = websiteUrl;
         this.staticResources = staticResources;
-        this.fileStorageService = fileStorageService;
 
         Spark.staticFiles.expireTime(600);
 
@@ -55,7 +49,6 @@ public class SearchService extends Service {
 
         Gson gson = GsonFactory.get();
 
-        Spark.get("/api/search", apiQueryService::apiSearch, gson::toJson);
         Spark.get("/public/search", searchQueryService::pathSearch);
         Spark.get("/public/site-search/:site/*", this::siteSearchRedir);
         Spark.get("/public/", frontPageService::render);

code/services-application/search-service/src/main/java/nu/marginalia/search/svc/SearchApiQueryService.java

@@ -1,108 +0,0 @@
-package nu.marginalia.search.svc;
-
-import com.google.common.base.Strings;
-import com.google.inject.Inject;
-import lombok.SneakyThrows;
-import nu.marginalia.db.DomainBlacklist;
-import nu.marginalia.index.client.model.results.SearchResultKeywordScore;
-import nu.marginalia.search.client.model.ApiSearchResultQueryDetails;
-import nu.marginalia.model.idx.WordMetadata;
-import nu.marginalia.search.SearchOperator;
-import nu.marginalia.search.model.UrlDetails;
-import nu.marginalia.search.client.model.ApiSearchResult;
-import nu.marginalia.search.client.model.ApiSearchResults;
-import nu.marginalia.search.model.SearchProfile;
-import nu.marginalia.client.Context;
-import nu.marginalia.search.command.SearchJsParameter;
-import nu.marginalia.search.model.UserSearchParameters;
-import spark.Request;
-import spark.Response;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class SearchApiQueryService {
-    private SearchOperator searchOperator;
-    private final DomainBlacklist blacklist;
-
-    @Inject
-    public SearchApiQueryService(
-            SearchOperator searchOperator,
-            DomainBlacklist blacklist
-            ) {
-        this.searchOperator = searchOperator;
-        this.blacklist = blacklist;
-    }
-
-    @SneakyThrows
-    public Object apiSearch(Request request, Response response) {
-
-        final var ctx = Context.fromRequest(request);
-        final String queryParam = request.queryParams("query");
-        final int limit;
-        SearchProfile profile = SearchProfile.YOLO;
-
-        String count = request.queryParamOrDefault("count", "20");
-        limit = Integer.parseInt(count);
-
-        String index = request.queryParamOrDefault("index", "0");
-        if (!Strings.isNullOrEmpty(index)) {
-            profile = switch (index) {
-                case "0" -> SearchProfile.YOLO;
-                case "1" -> SearchProfile.MODERN;
-                case "2" -> SearchProfile.DEFAULT;
-                case "3" -> SearchProfile.CORPO_CLEAN;
-                case "5" -> SearchProfile.YOLO;
-                case "blogosphere" -> SearchProfile.BLOGOSPHERE;
-                default -> SearchProfile.CORPO_CLEAN;
-            };
-        }
-
-        final String humanQuery = queryParam.trim();
-
-        var results = searchOperator.doApiSearch(ctx, new UserSearchParameters(humanQuery, profile, SearchJsParameter.DEFAULT));
-
-        results.removeIf(details -> blacklist.isBlacklisted(details.domainId));
-
-        return new ApiSearchResults("RESTRICTED", humanQuery, results.stream().map(this::convert).limit(limit).collect(Collectors.toList()));
-    }
-
-    ApiSearchResult convert(UrlDetails url) {
-        List<List<ApiSearchResultQueryDetails>> details = new ArrayList<>();
-        if (url.resultItem != null) {
-            var bySet = url.resultItem.keywordScores.stream().collect(Collectors.groupingBy(SearchResultKeywordScore::subquery));
-
-            outer:
-            for (var entries : bySet.values()) {
-                List<ApiSearchResultQueryDetails> lst = new ArrayList<>();
-                for (var entry : entries) {
-                    var metadata = new WordMetadata(entry.encodedWordMetadata());
-                    if (metadata.isEmpty())
-                        continue outer;
-
-                    Set<String> flags = metadata.flagSet().stream().map(Object::toString).collect(Collectors.toSet());
-                    lst.add(new ApiSearchResultQueryDetails(entry.keyword, Long.bitCount(metadata.positions()), flags));
-                }
-                details.add(lst);
-            }
-        }
-
-        return new ApiSearchResult(
-                url.url.toString(),
-                url.getTitle(),
-                url.getDescription(),
-                sanitizeNaN(url.getTermScore(), -100),
-                details
-        );
-    }
-
-    private double sanitizeNaN(double value, double alternative) {
-        if (!Double.isFinite(value)) {
-            return alternative;
-        }
-        return value;
-    }
-
-}

code/services-core/control-service/build.gradle

@@ -32,11 +32,12 @@ dependencies {
     implementation project(':code:libraries:message-queue')
     implementation project(':code:common:service-discovery')
     implementation project(':code:common:service-client')
-    implementation project(':code:api:search-api')
     implementation project(':code:api:index-api')
+    implementation project(':code:api:query-api')
     implementation project(':code:api:process-mqapi')
     implementation project(':code:features-search:screenshots')
     implementation project(':code:features-index:index-journal')
+    implementation project(':code:features-index:index-query')
     implementation project(':code:process-models:crawl-spec')
 
     implementation libs.bundles.slf4j

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlService.java

@@ -3,7 +3,6 @@ package nu.marginalia.control;
 import com.google.gson.Gson;
 import com.google.inject.Inject;
 import gnu.trove.list.array.TIntArrayList;
-import nu.marginalia.client.Context;
 import nu.marginalia.client.ServiceMonitors;
 import nu.marginalia.control.actor.Actor;
 import nu.marginalia.control.model.*;
@@ -11,11 +10,9 @@ import nu.marginalia.control.svc.*;
 import nu.marginalia.db.storage.model.FileStorageId;
 import nu.marginalia.db.storage.model.FileStorageType;
 import nu.marginalia.model.EdgeDomain;
-import nu.marginalia.model.EdgeUrl;
 import nu.marginalia.model.gson.GsonFactory;
 import nu.marginalia.renderer.RendererFactory;
 import nu.marginalia.screenshot.ScreenshotService;
-import nu.marginalia.search.client.SearchClient;
 import nu.marginalia.service.server.*;
 import org.eclipse.jetty.util.StringUtil;
 import org.slf4j.Logger;
@@ -40,8 +37,8 @@ public class ControlService extends Service {
     private final ApiKeyService apiKeyService;
     private final DomainComplaintService domainComplaintService;
     private final ControlBlacklistService blacklistService;
+    private final SearchToBanService searchToBanService;
     private final RandomExplorationService randomExplorationService;
-    private final SearchClient searchClient;
     private final ControlActorService controlActorService;
     private final StaticResources staticResources;
     private final MessageQueueService messageQueueService;
@@ -63,8 +60,8 @@ public class ControlService extends Service {
                           ControlBlacklistService blacklistService,
                           ControlActionsService controlActionsService,
                           ScreenshotService screenshotService,
-                          RandomExplorationService randomExplorationService,
+                          SearchToBanService searchToBanService,
-                          SearchClient searchClient
+                          RandomExplorationService randomExplorationService
                       ) throws IOException {
 
         super(params);
@@ -74,8 +71,8 @@ public class ControlService extends Service {
         this.apiKeyService = apiKeyService;
         this.domainComplaintService = domainComplaintService;
         this.blacklistService = blacklistService;
+        this.searchToBanService = searchToBanService;
         this.randomExplorationService = randomExplorationService;
-        this.searchClient = searchClient;
 
         var indexRenderer = rendererFactory.renderer("control/index");
         var eventsRenderer = rendererFactory.renderer("control/events");
@@ -176,8 +173,8 @@ public class ControlService extends Service {
         Spark.get("/public/blacklist", this::blacklistModel, blacklistRenderer::render);
         Spark.post("/public/blacklist", this::updateBlacklist, redirectToBlacklist);
 
-        Spark.get("/public/search-to-ban", this::searchToBanModel, searchToBanRenderer::render);
+        Spark.get("/public/search-to-ban", searchToBanService::handle, searchToBanRenderer::render);
-        Spark.post("/public/search-to-ban", this::searchToBanModel, searchToBanRenderer::render);
+        Spark.post("/public/search-to-ban", searchToBanService::handle, searchToBanRenderer::render);
 
         // API Keys
 
@@ -252,37 +249,6 @@ public class ControlService extends Service {
         return Map.of("blacklist", blacklistService.lastNAdditions(100));
     }
 
-    private Object searchToBanModel(Request request, Response response) {
-        String q = request.queryParams("q");
-
-        if (Objects.equals(request.requestMethod(), "POST")) {
-            request.params().forEach((k,v) -> System.out.println(k + " -- " + v));
-            List<String> bannedUrls = new ArrayList<>();
-
-            String query = request.queryParams("query");
-            for (var param : request.queryParams()) {
-                if ("query".equals(param)) {
-                    continue;
-                }
-                EdgeUrl.parse(param).ifPresent(url ->
-                        blacklistService.addToBlacklist(url.domain, query)
-                );
-                bannedUrls.add(param);
-            }
-
-            request.queryParams().forEach(System.out::println);
-            q = query;
-        }
-
-        if (q == null || q.isBlank()) {
-            return Map.of();
-        } else {
-            return searchClient
-                    .query(Context.fromRequest(request), q, 200, 5)
-                    .blockingFirst();
-        }
-    }
-
     private Object updateBlacklist(Request request, Response response) {
         var domain = new EdgeDomain(request.queryParams("domain"));
         if ("add".equals(request.queryParams("act"))) {

code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java

@@ -26,8 +26,6 @@ import nu.marginalia.mq.outbox.MqOutbox;
 import nu.marginalia.actor.prototype.AbstractActorPrototype;
 import nu.marginalia.actor.state.ActorState;
 import nu.marginalia.actor.state.ActorResumeBehavior;
-import nu.marginalia.search.client.SearchClient;
-import nu.marginalia.search.client.SearchMqEndpoints;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -57,7 +55,6 @@ public class ConvertAndLoadActor extends AbstractActorPrototype {
     private final MqOutbox mqLoaderOutbox;
     private final MqOutbox mqIndexConstructorOutbox;
     private final MqOutbox indexOutbox;
-    private final MqOutbox searchOutbox;
     private final FileStorageService storageService;
     private final BackupService backupService;
     private final Gson gson;
@@ -83,7 +80,6 @@ public class ConvertAndLoadActor extends AbstractActorPrototype {
                                ProcessOutboxes processOutboxes,
                                FileStorageService storageService,
                                IndexClient indexClient,
-                               SearchClient searchClient,
                                BackupService backupService,
                                Gson gson
                                    )
@@ -91,7 +87,6 @@ public class ConvertAndLoadActor extends AbstractActorPrototype {
         super(stateFactory);
         this.processWatcher = processWatcher;
         this.indexOutbox = indexClient.outbox();
-        this.searchOutbox = searchClient.outbox();
         this.mqConverterOutbox = processOutboxes.getConverterOutbox();
         this.mqLoaderOutbox = processOutboxes.getLoaderOutbox();
         this.mqIndexConstructorOutbox = processOutboxes.getIndexConstructorOutbox();

code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlBlacklistService.java

@@ -4,6 +4,8 @@ import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.control.model.BlacklistedDomainModel;
 import nu.marginalia.model.EdgeDomain;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -12,6 +14,7 @@ import java.util.List;
 public class ControlBlacklistService {
 
     private final HikariDataSource dataSource;
+    private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Inject
     public ControlBlacklistService(HikariDataSource dataSource) {
@@ -19,6 +22,8 @@ public class ControlBlacklistService {
     }
 
     public void addToBlacklist(EdgeDomain domain, String comment) {
+        logger.info("Blacklisting {} -- {}", domain, comment);
+
         try (var conn = dataSource.getConnection();
              var stmt = conn.prepareStatement("""
                      INSERT IGNORE INTO EC_DOMAIN_BLACKLIST (URL_DOMAIN, COMMENT) VALUES (?, ?)
@@ -33,6 +38,8 @@ public class ControlBlacklistService {
     }
 
     public void removeFromBlacklist(EdgeDomain domain) {
+        logger.info("Un-blacklisting {}", domain);
+
         try (var conn = dataSource.getConnection();
              var stmt = conn.prepareStatement("""
                      DELETE FROM EC_DOMAIN_BLACKLIST WHERE URL_DOMAIN=?

code/services-core/control-service/src/main/java/nu/marginalia/control/svc/SearchToBanService.java

@@ -0,0 +1,69 @@
+package nu.marginalia.control.svc;
+
+import com.google.inject.Inject;
+import nu.marginalia.client.Context;
+import nu.marginalia.index.client.model.query.SearchSetIdentifier;
+import nu.marginalia.index.query.limit.QueryLimits;
+import nu.marginalia.model.EdgeUrl;
+import nu.marginalia.query.client.QueryClient;
+import nu.marginalia.query.model.QueryParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import spark.Request;
+import spark.Response;
+
+import java.util.Map;
+import java.util.Objects;
+
+public class SearchToBanService {
+    private final ControlBlacklistService blacklistService;
+    private final QueryClient queryClient;
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    @Inject
+    public SearchToBanService(ControlBlacklistService blacklistService,
+                              QueryClient queryClient)
+    {
+
+        this.blacklistService = blacklistService;
+        this.queryClient = queryClient;
+    }
+
+    public Object handle(Request request, Response response) {
+        if (Objects.equals(request.requestMethod(), "POST")) {
+            executeBlacklisting(request);
+
+            return findResults(Context.fromRequest(request), request.queryParams("query"));
+        }
+
+        return findResults(Context.fromRequest(request), request.queryParams("q"));
+    }
+
+    private Object findResults(Context ctx, String q) {
+        if (q == null || q.isBlank()) {
+            return Map.of();
+        } else {
+            return executeQuery(ctx, q);
+        }
+    }
+
+    private void executeBlacklisting(Request request) {
+        String query = request.queryParams("query");
+        for (var param : request.queryParams()) {
+            logger.info(param + ": " + request.queryParams(param));
+            if ("query".equals(param)) {
+                continue;
+            }
+            EdgeUrl.parse(param).ifPresent(url ->
+                    blacklistService.addToBlacklist(url.domain, query)
+            );
+        }
+    }
+
+    private Object executeQuery(Context ctx, String query) {
+        return queryClient.search(ctx, new QueryParams(
+                query, new QueryLimits(2, 200, 250, 8192),
+                SearchSetIdentifier.NONE
+        ));
+    }
+}

code/services-core/control-service/src/main/resources/templates/control/search-to-ban.hdb

@@ -11,7 +11,7 @@
     <h1>Search to Ban</h1>
     <form action="/search-to-ban">
         <label for="q">Search Query</label><br>
-        <input type="text" value="{{query}}" name="q" id="q" /> <input type="submit" value="Search" />
+        <input type="text" value="{{specs.humanQuery}}" name="q" id="q" /> <input type="submit" value="Search" />
     </form>
     {{#unless results}}
     <p>This utility lets you use the search engine to find spammy results, and ban them
@@ -21,7 +21,7 @@
     <hr>
     <form action="/search-to-ban" method="post">
     <input type="submit" value="Blacklist Selected Domains">
-    <input type="hidden" name="query" text="{{query}}" />
+    <input type="hidden" name="query" value="{{specs.humanQuery}}" />
     <table>
     {{#each results}}
         <tr><td><input type="checkbox" name="{{url}}"><td>{{title}}</td></tr>

settings.gradle

@@ -45,7 +45,6 @@ include 'code:features-index:index-forward'
 include 'code:features-index:index-reverse'
 include 'code:features-index:domain-ranking'
 
-include 'code:api:search-api'
 include 'code:api:query-api'
 include 'code:api:index-api'
 include 'code:api:assistant-api'