(query-client) Add support for fault-tolerant requests to single node services

Adding a method importantCall that will retry a failing request on each route until it succeeds or the routes run out.

code/api/query-api/src/main/java/nu/marginalia/query/client/QueryClient.java

@@ -47,10 +47,11 @@ public class QueryClient  {
 
     @CheckReturnValue
     public QueryResponse search(QueryParams params) {
+        var query = QueryProtobufCodec.convertQueryParams(params);
+
         return wmsa_qs_api_search_time.time(
                 () ->  QueryProtobufCodec.convertQueryResponse(queryApiPool
-                        .api()
+                        .importantCall((api) -> api.query(query))
-                        .query(QueryProtobufCodec.convertQueryParams(params))
                 )
         );
     }

code/common/service-discovery/src/main/java/nu/marginalia/service/client/GrpcSingleNodeChannelPool.java

@@ -98,6 +98,22 @@ public class GrpcSingleNodeChannelPool<STUB> extends ServiceChangeMonitor {
         return stubConstructor.apply(getChannel());
     }
 
+    /** Try to make the call go through.  The function will cycle through
+     * available routes until exhaustion, and only then will it give up
+     */
+    public <T> T importantCall(Function<STUB, T> function) {
+        for (int i = 0; i < channels.size(); i++) {
+            try {
+                return function.apply(api());
+            }
+            catch (Exception e) {
+                logger.error("API Exception", e);
+            }
+        }
+
+        throw new ServiceNotAvailableException(serviceId);
+    }
+
     /** Get the channel that is most ready to use */
     public ManagedChannel getChannel() {
         return channels