(search) Add site subscription feature that puts RSS updates on the front page

code/services-application/search-service/java/nu/marginalia/search/svc/SearchFrontPageService.java

@@ -2,73 +2,99 @@ package nu.marginalia.search.svc;
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
-import com.zaxxer.hikari.HikariDataSource;
+import io.jooby.Context;
 import io.jooby.MapModelAndView;
 import io.jooby.annotation.GET;
 import io.jooby.annotation.Path;
 import nu.marginalia.WebsiteUrl;
+import nu.marginalia.api.feeds.FeedsClient;
+import nu.marginalia.api.feeds.RpcFeed;
 import nu.marginalia.search.model.NavbarModel;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.sql.SQLException;
-import java.time.LocalDate;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 
 /** Renders the front page (index) */
 @Singleton
 public class SearchFrontPageService {
 
-    private final HikariDataSource dataSource;
     private final SearchQueryCountService searchVisitorCount;
+    private final FeedsClient feedsClient;
     private final WebsiteUrl websiteUrl;
+    private final SearchSiteSubscriptionService  subscriptionService;
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Inject
-    public SearchFrontPageService(HikariDataSource dataSource,
+    public SearchFrontPageService(SearchQueryCountService searchVisitorCount,
-                                  SearchQueryCountService searchVisitorCount,
+                                  FeedsClient feedsClient,
-                                  WebsiteUrl websiteUrl
+                                  WebsiteUrl websiteUrl,
+                                  SearchSiteSubscriptionService subscriptionService
     ) {
-        this.dataSource = dataSource;
         this.searchVisitorCount = searchVisitorCount;
+        this.feedsClient = feedsClient;
         this.websiteUrl = websiteUrl;
+        this.subscriptionService = subscriptionService;
     }
 
     @GET
     @Path("/")
-    public MapModelAndView render() {
+    public MapModelAndView render(Context context) {
+
+        List<NewsItem> newsItems = getNewsItems(context);
+
+        IndexModel model = new IndexModel(newsItems, searchVisitorCount.getQueriesPerMinute());
+
         return new MapModelAndView("serp/start.jte")
                 .put("navbar", NavbarModel.SEARCH)
+                .put("model", model)
                 .put("websiteUrl", websiteUrl);
     }
 
+    private List<NewsItem> getNewsItems(Context context) {
 
-    private List<NewsItem> getNewsItems() {
+        Set<Integer> subscriptions = subscriptionService.getSubscriptions(context);
-        List<NewsItem> items = new ArrayList<>();
 
-        try (var conn = dataSource.getConnection();
+        if (subscriptions.isEmpty())
-             var stmt = conn.prepareStatement("""
+            return List.of();
-                SELECT TITLE, LINK, SOURCE, LIST_DATE FROM SEARCH_NEWS_FEED ORDER BY LIST_DATE DESC
-                """)) {
 
-            var rep = stmt.executeQuery();
+        List<CompletableFuture<RpcFeed>> feedResults = new ArrayList<>();
 
-            while (rep.next()) {
+        for (int sub : subscriptions) {
-                items.add(new NewsItem(
+            feedResults.add(feedsClient.getFeed(sub));
-                        rep.getString(1),
-                        rep.getString(2),
-                        rep.getString(3),
-                        rep.getDate(4).toLocalDate()));
-            }
-        }
-        catch (SQLException ex) {
-            logger.warn("Failed to fetch news items", ex);
         }
 
-        return items;
+        List<NewsItem> ret = new ArrayList<>();
+        for (var result : feedResults) {
+            try {
+                RpcFeed feed = result.get();
+
+                for (var item : feed.getItemsList()) {
+                    String title = item.getTitle();
+                    if (title.isBlank()) {
+                        title = "[Missing Title]";
                     }
+                    ret.add(new NewsItem(title, item.getUrl(), feed.getDomain(), item.getDescription(), item.getDate()));
+                }
+            }
+            catch (Exception ex) {
+                logger.error("Failed to fetch news items", ex);
+            }
+        }
+
+        ret.sort(Comparator.comparing(NewsItem::date).reversed());
+        if (ret.size() > 25) {
+            ret.subList(25, ret.size()).clear();
+        }
+        return ret;
+    }
+
+
 
     /* FIXME
     public Object renderNewsFeed(Request request, Response response) {
@@ -108,6 +134,6 @@ public class SearchFrontPageService {
         return sb.toString();
     }*/
 
-    private record IndexModel(List<NewsItem> news, int searchPerMinute) { }
+    public record IndexModel(List<NewsItem> news, int searchPerMinute) { }
-    private record NewsItem(String title, String url, String source, LocalDate date) {}
+    public record NewsItem(String title, String url, String domain, String description, String date) {}
 }

code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java

@@ -2,6 +2,7 @@ package nu.marginalia.search.svc;
 
 import com.google.inject.Inject;
 import com.zaxxer.hikari.HikariDataSource;
+import io.jooby.Context;
 import io.jooby.MapModelAndView;
 import io.jooby.ModelAndView;
 import io.jooby.annotation.*;
@@ -15,7 +16,6 @@ import nu.marginalia.api.livecapture.LiveCaptureClient;
 import nu.marginalia.db.DbDomainQueries;
 import nu.marginalia.model.EdgeDomain;
 import nu.marginalia.screenshot.ScreenshotService;
-import nu.marginalia.search.JteRenderer;
 import nu.marginalia.search.SearchOperator;
 import nu.marginalia.search.model.GroupedUrlDetails;
 import nu.marginalia.search.model.NavbarModel;
@@ -44,7 +44,7 @@ public class SearchSiteInfoService {
     private final ScreenshotService screenshotService;
 
     private final HikariDataSource dataSource;
-    private final JteRenderer jteRenderer;
+    private final SearchSiteSubscriptionService searchSiteSubscriptions;
 
     @Inject
     public SearchSiteInfoService(SearchOperator searchOperator,
@@ -55,7 +55,7 @@ public class SearchSiteInfoService {
                                  LiveCaptureClient liveCaptureClient,
                                  ScreenshotService screenshotService,
                                  HikariDataSource dataSource,
-                                 JteRenderer jteRenderer)
+                                 SearchSiteSubscriptionService searchSiteSubscriptions)
     {
         this.searchOperator = searchOperator;
         this.domainInfoClient = domainInfoClient;
@@ -66,7 +66,7 @@ public class SearchSiteInfoService {
         this.liveCaptureClient = liveCaptureClient;
         this.screenshotService = screenshotService;
         this.dataSource = dataSource;
-        this.jteRenderer = jteRenderer;
+        this.searchSiteSubscriptions = searchSiteSubscriptions;
     }
 
     @GET
@@ -103,6 +103,7 @@ public class SearchSiteInfoService {
     @GET
     @Path("/site/{domainName}")
     public ModelAndView<?>  handle(
+            Context context,
             @PathParam String domainName,
             @QueryParam String view,
             @QueryParam Integer page
@@ -118,15 +119,23 @@ public class SearchSiteInfoService {
         SiteInfoModel model = switch (view) {
             case "links" -> listLinks(domainName, page);
             case "docs" -> listDocs(domainName, page);
-            case "info" -> listInfo(domainName);
+            case "info" -> listInfo(context, domainName);
             case "report" -> reportSite(domainName);
-            default -> listInfo(domainName);
+            default -> listInfo(context, domainName);
         };
 
         return new MapModelAndView("siteinfo/main.jte",
                 Map.of("model", model, "navbar", NavbarModel.SITEINFO));
     }
 
+    @POST
+    @Path("/site/{domainName}/subscribe")
+    public ModelAndView<?> toggleSubscription(Context context, @PathParam String domainName) throws SQLException {
+        searchSiteSubscriptions.toggleSubscription(context, new EdgeDomain(domainName));
+
+        return new MapModelAndView("redirect.jte", Map.of("url", "/site/"+domainName));
+    }
+
     @POST
     @Path("/site/{domainName}")
     public ModelAndView<?> handleComplaint(
@@ -184,7 +193,7 @@ public class SearchSiteInfoService {
         );
     }
 
-    private SiteInfoWithContext listInfo(String domainName) {
+    private SiteInfoWithContext listInfo(Context context, String domainName) {
 
         var domain = new EdgeDomain(domainName);
         final int domainId = domainQueries.tryGetDomainId(domain).orElse(-1);
@@ -198,6 +207,7 @@ public class SearchSiteInfoService {
 
         boolean hasScreenshot = screenshotService.hasScreenshot(domainId);
 
+        boolean isSubscribed = searchSiteSubscriptions.isSubscribed(context, domain);
 
         if (domainId < 0) {
             domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID"));
@@ -224,6 +234,7 @@ public class SearchSiteInfoService {
         }
 
         var result = new SiteInfoWithContext(domainName,
+                isSubscribed,
                 viableAliasDomain ? domain.aliasDomain().map(EdgeDomain::toString) : Optional.empty(),
                 domainId,
                 url,
@@ -341,6 +352,7 @@ public class SearchSiteInfoService {
     }
 
     public record SiteInfoWithContext(String domain,
+                                      boolean isSubscribed,
                                       Optional<String> aliasDomain,
                                       int domainId,
                                       String siteUrl,

code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteSubscriptionService.java

@@ -0,0 +1,87 @@
+package nu.marginalia.search.svc;
+
+import com.google.inject.Inject;
+import io.jooby.Context;
+import io.jooby.Cookie;
+import io.jooby.Value;
+import nu.marginalia.db.DbDomainQueries;
+import nu.marginalia.model.EdgeDomain;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+import java.time.Duration;
+import java.util.Base64;
+import java.util.HashSet;
+import java.util.Set;
+
+public class SearchSiteSubscriptionService {
+    private final DbDomainQueries dbDomainQueries;
+
+    private static final Logger logger = LoggerFactory.getLogger(SearchSiteSubscriptionService.class);
+
+    @Inject
+    public SearchSiteSubscriptionService(DbDomainQueries dbDomainQueries) {
+        this.dbDomainQueries = dbDomainQueries;
+    }
+
+    public HashSet<Integer> getSubscriptions(Context context) {
+        Value cookieValue = context.cookie("sub");
+        if (cookieValue.isPresent()) {
+            return decodeSubscriptionsCookie(cookieValue.value());
+        }
+        else {
+            return new HashSet<>();
+        }
+    }
+
+    public void putSubscriptions(Context context, Set<Integer> values) {
+        var cookie = new Cookie("sub", encodeSubscriptionsCookie(values));
+        cookie.setMaxAge(Duration.ofDays(365));
+        context.setResponseCookie(cookie);
+    }
+
+    private HashSet<Integer> decodeSubscriptionsCookie(String encodedValue) {
+        if (encodedValue == null || encodedValue.isEmpty())
+            return new HashSet<>();
+        IntBuffer buffer = ByteBuffer.wrap(Base64.getDecoder().decode(encodedValue)).asIntBuffer();
+        HashSet<Integer> ret = new HashSet<>(buffer.capacity());
+        while (buffer.hasRemaining())
+            ret.add(buffer.get());
+        return ret;
+    }
+
+    private String encodeSubscriptionsCookie(Set<Integer> subscriptions) {
+        if (subscriptions.isEmpty())
+            return "";
+
+        byte[] bytes = new byte[4 * subscriptions.size()];
+        IntBuffer buffer = ByteBuffer.wrap(bytes).asIntBuffer();
+        for (int val : subscriptions) {
+            buffer.put(val);
+        }
+        return Base64.getEncoder().encodeToString(bytes);
+    }
+
+    public boolean isSubscribed(Context context, EdgeDomain domain) {
+        int domainId = dbDomainQueries.getDomainId(domain);
+
+        return getSubscriptions(context).contains(domainId);
+    }
+
+    public void toggleSubscription(Context context, EdgeDomain domain) {
+
+        Set<Integer> subscriptions = getSubscriptions(context);
+        int domainId = dbDomainQueries.getDomainId(domain);
+
+        if (subscriptions.contains(domainId)) {
+            subscriptions.remove(domainId);
+        }
+        else {
+            subscriptions.add(domainId);
+        }
+
+        putSubscriptions(context, subscriptions);
+    }
+}

code/services-application/search-service/resources/jte/serp/start.jte

@@ -2,9 +2,12 @@
 @import nu.marginalia.search.model.NavbarModel
 @import nu.marginalia.search.model.SearchFilters
 @import nu.marginalia.search.model.SearchProfile
+@import nu.marginalia.search.svc.SearchFrontPageService.IndexModel
+@import nu.marginalia.search.svc.SearchFrontPageService.NewsItem
 
 @param NavbarModel navbar
 @param WebsiteUrl websiteUrl
+@param IndexModel model
 
 <!DOCTYPE html>
 <html lang="en">
@@ -26,6 +29,7 @@
     </div>
 </header>
 
+@if (model.news().isEmpty())
 <div class="max-w-7xl mx-auto flex flex-col space-y-4 fill-w">
     <div class="border dark:border-gray-600 dark:bg-gray-800 bg-white rounded p-2 m-4 ">
         <div class="text-slate-700 dark:text-white text-sm p-4">
@@ -74,6 +78,28 @@
     </div>
 
 </div>
+@else
+<div class="max-w-7xl mx-auto flex flex-col space-y-4 fill-w m-4">
+    <div class="my-4 text-black dark:text-white font-serif text-xl mx-8">Subscriptions</div>
+    @for (NewsItem item : model.news())
+        <div class="border dark:border-gray-600 rounded bg-white dark:bg-gray-800 flex flex-col overflow-hidden mx-4 p-4 space-y-2">
+            <div class="text-black dark:text-white flex place-items-middle">
+                <div class="flex flex-col flex-col space-y-1">
+                    <a class="text-l text-liteblue dark:text-blue-200" href="${item.url()}" rel="ugc external nofollow">${item.title()}</a>
+                    <a class="text-xs text-gray-800 dark:text-gray-100" href="/site/${item.domain()}">
+                        <i class="fas fa-globe"></i> ${item.domain()}
+                    </a>
+                </div>
+                <div class="grow"></div>
+                <div class="flex text-xs">
+                    ${item.date().substring(0, 10)}
+                </div>
+            </div>
+            <div class="text-sm text-gray-800 dark:text-gray-100">$unsafe{item.description()}</div>
+        </div>
+    @endfor
+</div>
+@endif
 
 @template.part.footerLegal()
 <script lang="javascript" src="js/typeahead.js"></script>

code/services-application/search-service/resources/jte/siteinfo/view/overview.jte

@@ -32,7 +32,19 @@
                 <i class="fas fa-rss text-orange-500"></i>
                 <span class="grow">Feed</span>
 
-                <i class="fa-regular fa-bookmark mr-2 text-gray-600 hover:text-blue-800 dark:text-gray-100 dark:hover:text-gray-400 cursor-pointer" title="Add content to this feed to the front page"></i>
+                <form method="post" action="/site/${siteInfo.domain()}/subscribe">
+                @if (siteInfo.isSubscribed())
+                    <button type="submit">
+                    <i class="fa-solid fa-bookmark mr-2 text-blue-600 hover:text-blue-800 dark:text-blue-100 dark:hover:text-blue-400 cursor-pointer"
+                        title="Unsubscribe from the front page"></i>
+                    </button>
+                @else
+                    <button type="submit">
+                        <i class="fa-regular fa-bookmark mr-2 text-gray-600 hover:text-blue-800 dark:text-gray-100 dark:hover:text-gray-400 cursor-pointer"
+                           title="Add content to this feed to the front page"></i>
+                    </button>
+                @endif
+                </form>
             </div>
 
             <dl class="mx-3 text-gray-800 dark:text-white">