From 9f70cecaefeb2b779dfd72e9e131227b48578058 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 17 Dec 2024 21:40:52 +0100 Subject: [PATCH] (search) Add site subscription feature that puts RSS updates on the front page --- .../search/svc/SearchFrontPageService.java | 82 +++++++++++------ .../search/svc/SearchSiteInfoService.java | 26 ++++-- .../svc/SearchSiteSubscriptionService.java | 87 +++++++++++++++++++ .../resources/jte/serp/start.jte | 26 ++++++ .../resources/jte/siteinfo/view/overview.jte | 14 ++- 5 files changed, 199 insertions(+), 36 deletions(-) create mode 100644 code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteSubscriptionService.java diff --git a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchFrontPageService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchFrontPageService.java index de00d6e9..b2154897 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchFrontPageService.java +++ b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchFrontPageService.java @@ -2,74 +2,100 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.zaxxer.hikari.HikariDataSource; +import io.jooby.Context; import io.jooby.MapModelAndView; import io.jooby.annotation.GET; import io.jooby.annotation.Path; import nu.marginalia.WebsiteUrl; +import nu.marginalia.api.feeds.FeedsClient; +import nu.marginalia.api.feeds.RpcFeed; import nu.marginalia.search.model.NavbarModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.SQLException; -import java.time.LocalDate; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; /** Renders the front page (index) */ @Singleton public class SearchFrontPageService { - private final HikariDataSource dataSource; private final SearchQueryCountService searchVisitorCount; + private final FeedsClient feedsClient; private final WebsiteUrl websiteUrl; + private final SearchSiteSubscriptionService subscriptionService; private final Logger logger = LoggerFactory.getLogger(getClass()); @Inject - public SearchFrontPageService(HikariDataSource dataSource, - SearchQueryCountService searchVisitorCount, - WebsiteUrl websiteUrl + public SearchFrontPageService(SearchQueryCountService searchVisitorCount, + FeedsClient feedsClient, + WebsiteUrl websiteUrl, + SearchSiteSubscriptionService subscriptionService ) { - this.dataSource = dataSource; this.searchVisitorCount = searchVisitorCount; + this.feedsClient = feedsClient; this.websiteUrl = websiteUrl; + this.subscriptionService = subscriptionService; } @GET @Path("/") - public MapModelAndView render() { + public MapModelAndView render(Context context) { + + List newsItems = getNewsItems(context); + + IndexModel model = new IndexModel(newsItems, searchVisitorCount.getQueriesPerMinute()); + return new MapModelAndView("serp/start.jte") .put("navbar", NavbarModel.SEARCH) + .put("model", model) .put("websiteUrl", websiteUrl); } + private List getNewsItems(Context context) { - private List getNewsItems() { - List items = new ArrayList<>(); + Set subscriptions = subscriptionService.getSubscriptions(context); - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement(""" - SELECT TITLE, LINK, SOURCE, LIST_DATE FROM SEARCH_NEWS_FEED ORDER BY LIST_DATE DESC - """)) { + if (subscriptions.isEmpty()) + return List.of(); - var rep = stmt.executeQuery(); + List> feedResults = new ArrayList<>(); - while (rep.next()) { - items.add(new NewsItem( - rep.getString(1), - rep.getString(2), - rep.getString(3), - rep.getDate(4).toLocalDate())); + for (int sub : subscriptions) { + feedResults.add(feedsClient.getFeed(sub)); + } + + List ret = new ArrayList<>(); + for (var result : feedResults) { + try { + RpcFeed feed = result.get(); + + for (var item : feed.getItemsList()) { + String title = item.getTitle(); + if (title.isBlank()) { + title = "[Missing Title]"; + } + ret.add(new NewsItem(title, item.getUrl(), feed.getDomain(), item.getDescription(), item.getDate())); + } + } + catch (Exception ex) { + logger.error("Failed to fetch news items", ex); } } - catch (SQLException ex) { - logger.warn("Failed to fetch news items", ex); - } - return items; + ret.sort(Comparator.comparing(NewsItem::date).reversed()); + if (ret.size() > 25) { + ret.subList(25, ret.size()).clear(); + } + return ret; } + + /* FIXME public Object renderNewsFeed(Request request, Response response) { List newsItems = getNewsItems(); @@ -108,6 +134,6 @@ public class SearchFrontPageService { return sb.toString(); }*/ - private record IndexModel(List news, int searchPerMinute) { } - private record NewsItem(String title, String url, String source, LocalDate date) {} + public record IndexModel(List news, int searchPerMinute) { } + public record NewsItem(String title, String url, String domain, String description, String date) {} } diff --git a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java index d9acebdf..3eacc95d 100644 --- a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java +++ b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteInfoService.java @@ -2,6 +2,7 @@ package nu.marginalia.search.svc; import com.google.inject.Inject; import com.zaxxer.hikari.HikariDataSource; +import io.jooby.Context; import io.jooby.MapModelAndView; import io.jooby.ModelAndView; import io.jooby.annotation.*; @@ -15,7 +16,6 @@ import nu.marginalia.api.livecapture.LiveCaptureClient; import nu.marginalia.db.DbDomainQueries; import nu.marginalia.model.EdgeDomain; import nu.marginalia.screenshot.ScreenshotService; -import nu.marginalia.search.JteRenderer; import nu.marginalia.search.SearchOperator; import nu.marginalia.search.model.GroupedUrlDetails; import nu.marginalia.search.model.NavbarModel; @@ -44,7 +44,7 @@ public class SearchSiteInfoService { private final ScreenshotService screenshotService; private final HikariDataSource dataSource; - private final JteRenderer jteRenderer; + private final SearchSiteSubscriptionService searchSiteSubscriptions; @Inject public SearchSiteInfoService(SearchOperator searchOperator, @@ -55,7 +55,7 @@ public class SearchSiteInfoService { LiveCaptureClient liveCaptureClient, ScreenshotService screenshotService, HikariDataSource dataSource, - JteRenderer jteRenderer) + SearchSiteSubscriptionService searchSiteSubscriptions) { this.searchOperator = searchOperator; this.domainInfoClient = domainInfoClient; @@ -66,7 +66,7 @@ public class SearchSiteInfoService { this.liveCaptureClient = liveCaptureClient; this.screenshotService = screenshotService; this.dataSource = dataSource; - this.jteRenderer = jteRenderer; + this.searchSiteSubscriptions = searchSiteSubscriptions; } @GET @@ -103,6 +103,7 @@ public class SearchSiteInfoService { @GET @Path("/site/{domainName}") public ModelAndView handle( + Context context, @PathParam String domainName, @QueryParam String view, @QueryParam Integer page @@ -118,15 +119,23 @@ public class SearchSiteInfoService { SiteInfoModel model = switch (view) { case "links" -> listLinks(domainName, page); case "docs" -> listDocs(domainName, page); - case "info" -> listInfo(domainName); + case "info" -> listInfo(context, domainName); case "report" -> reportSite(domainName); - default -> listInfo(domainName); + default -> listInfo(context, domainName); }; return new MapModelAndView("siteinfo/main.jte", Map.of("model", model, "navbar", NavbarModel.SITEINFO)); } + @POST + @Path("/site/{domainName}/subscribe") + public ModelAndView toggleSubscription(Context context, @PathParam String domainName) throws SQLException { + searchSiteSubscriptions.toggleSubscription(context, new EdgeDomain(domainName)); + + return new MapModelAndView("redirect.jte", Map.of("url", "/site/"+domainName)); + } + @POST @Path("/site/{domainName}") public ModelAndView handleComplaint( @@ -184,7 +193,7 @@ public class SearchSiteInfoService { ); } - private SiteInfoWithContext listInfo(String domainName) { + private SiteInfoWithContext listInfo(Context context, String domainName) { var domain = new EdgeDomain(domainName); final int domainId = domainQueries.tryGetDomainId(domain).orElse(-1); @@ -198,6 +207,7 @@ public class SearchSiteInfoService { boolean hasScreenshot = screenshotService.hasScreenshot(domainId); + boolean isSubscribed = searchSiteSubscriptions.isSubscribed(context, domain); if (domainId < 0) { domainInfoFuture = CompletableFuture.failedFuture(new Exception("Unknown Domain ID")); @@ -224,6 +234,7 @@ public class SearchSiteInfoService { } var result = new SiteInfoWithContext(domainName, + isSubscribed, viableAliasDomain ? domain.aliasDomain().map(EdgeDomain::toString) : Optional.empty(), domainId, url, @@ -341,6 +352,7 @@ public class SearchSiteInfoService { } public record SiteInfoWithContext(String domain, + boolean isSubscribed, Optional aliasDomain, int domainId, String siteUrl, diff --git a/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteSubscriptionService.java b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteSubscriptionService.java new file mode 100644 index 00000000..a7143185 --- /dev/null +++ b/code/services-application/search-service/java/nu/marginalia/search/svc/SearchSiteSubscriptionService.java @@ -0,0 +1,87 @@ +package nu.marginalia.search.svc; + +import com.google.inject.Inject; +import io.jooby.Context; +import io.jooby.Cookie; +import io.jooby.Value; +import nu.marginalia.db.DbDomainQueries; +import nu.marginalia.model.EdgeDomain; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.time.Duration; +import java.util.Base64; +import java.util.HashSet; +import java.util.Set; + +public class SearchSiteSubscriptionService { + private final DbDomainQueries dbDomainQueries; + + private static final Logger logger = LoggerFactory.getLogger(SearchSiteSubscriptionService.class); + + @Inject + public SearchSiteSubscriptionService(DbDomainQueries dbDomainQueries) { + this.dbDomainQueries = dbDomainQueries; + } + + public HashSet getSubscriptions(Context context) { + Value cookieValue = context.cookie("sub"); + if (cookieValue.isPresent()) { + return decodeSubscriptionsCookie(cookieValue.value()); + } + else { + return new HashSet<>(); + } + } + + public void putSubscriptions(Context context, Set values) { + var cookie = new Cookie("sub", encodeSubscriptionsCookie(values)); + cookie.setMaxAge(Duration.ofDays(365)); + context.setResponseCookie(cookie); + } + + private HashSet decodeSubscriptionsCookie(String encodedValue) { + if (encodedValue == null || encodedValue.isEmpty()) + return new HashSet<>(); + IntBuffer buffer = ByteBuffer.wrap(Base64.getDecoder().decode(encodedValue)).asIntBuffer(); + HashSet ret = new HashSet<>(buffer.capacity()); + while (buffer.hasRemaining()) + ret.add(buffer.get()); + return ret; + } + + private String encodeSubscriptionsCookie(Set subscriptions) { + if (subscriptions.isEmpty()) + return ""; + + byte[] bytes = new byte[4 * subscriptions.size()]; + IntBuffer buffer = ByteBuffer.wrap(bytes).asIntBuffer(); + for (int val : subscriptions) { + buffer.put(val); + } + return Base64.getEncoder().encodeToString(bytes); + } + + public boolean isSubscribed(Context context, EdgeDomain domain) { + int domainId = dbDomainQueries.getDomainId(domain); + + return getSubscriptions(context).contains(domainId); + } + + public void toggleSubscription(Context context, EdgeDomain domain) { + + Set subscriptions = getSubscriptions(context); + int domainId = dbDomainQueries.getDomainId(domain); + + if (subscriptions.contains(domainId)) { + subscriptions.remove(domainId); + } + else { + subscriptions.add(domainId); + } + + putSubscriptions(context, subscriptions); + } +} diff --git a/code/services-application/search-service/resources/jte/serp/start.jte b/code/services-application/search-service/resources/jte/serp/start.jte index 0af275cd..0e4b97c0 100644 --- a/code/services-application/search-service/resources/jte/serp/start.jte +++ b/code/services-application/search-service/resources/jte/serp/start.jte @@ -2,9 +2,12 @@ @import nu.marginalia.search.model.NavbarModel @import nu.marginalia.search.model.SearchFilters @import nu.marginalia.search.model.SearchProfile +@import nu.marginalia.search.svc.SearchFrontPageService.IndexModel +@import nu.marginalia.search.svc.SearchFrontPageService.NewsItem @param NavbarModel navbar @param WebsiteUrl websiteUrl +@param IndexModel model @@ -26,6 +29,7 @@ +@if (model.news().isEmpty())
@@ -74,6 +78,28 @@
+@else +
+
Subscriptions
+ @for (NewsItem item : model.news()) +
+
+
+ ${item.title()} + + ${item.domain()} + +
+
+
+ ${item.date().substring(0, 10)} +
+
+
$unsafe{item.description()}
+
+ @endfor +
+@endif @template.part.footerLegal() diff --git a/code/services-application/search-service/resources/jte/siteinfo/view/overview.jte b/code/services-application/search-service/resources/jte/siteinfo/view/overview.jte index f5c568f1..e12c5b8b 100644 --- a/code/services-application/search-service/resources/jte/siteinfo/view/overview.jte +++ b/code/services-application/search-service/resources/jte/siteinfo/view/overview.jte @@ -32,7 +32,19 @@ Feed - +
+ @if (siteInfo.isSubscribed()) + + @else + + @endif +