From fbba3924911c9440901de65f80b7d585f9557932 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Tue, 4 Feb 2025 13:36:49 +0100 Subject: [PATCH] (live-capture) Send a UA-string from the browserless fetcher as well The change also introduces a somewhat convoluted wiremock test to intercept and verify that these headers are in fact sent --- code/functions/live-capture/build.gradle | 1 + .../livecapture/BrowserlessClient.java | 5 ++ .../livecapture/BrowserlessClientTest.java | 62 ++++++++++++++++++- settings.gradle | 1 + 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/code/functions/live-capture/build.gradle b/code/functions/live-capture/build.gradle index 4d0dc4df..b5b93fb7 100644 --- a/code/functions/live-capture/build.gradle +++ b/code/functions/live-capture/build.gradle @@ -34,6 +34,7 @@ dependencies { implementation libs.bundles.slf4j implementation libs.commons.lang3 implementation libs.commons.io + implementation libs.wiremock implementation libs.prometheus implementation libs.guava diff --git a/code/functions/live-capture/java/nu/marginalia/livecapture/BrowserlessClient.java b/code/functions/live-capture/java/nu/marginalia/livecapture/BrowserlessClient.java index ae27cf65..7bb7c0d5 100644 --- a/code/functions/live-capture/java/nu/marginalia/livecapture/BrowserlessClient.java +++ b/code/functions/live-capture/java/nu/marginalia/livecapture/BrowserlessClient.java @@ -1,6 +1,7 @@ package nu.marginalia.livecapture; import com.google.gson.Gson; +import nu.marginalia.WmsaHome; import nu.marginalia.model.gson.GsonFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +28,8 @@ public class BrowserlessClient implements AutoCloseable { private final URI browserlessURI; private final Gson gson = GsonFactory.get(); + private final String userAgent = WmsaHome.getUserAgent().uaString(); + public BrowserlessClient(URI browserlessURI) { this.browserlessURI = browserlessURI; } @@ -34,6 +37,7 @@ public class BrowserlessClient implements AutoCloseable { public String content(String url, GotoOptions gotoOptions) throws IOException, InterruptedException { Map requestData = Map.of( "url", url, + "userAgent", userAgent, "gotoOptions", gotoOptions ); @@ -60,6 +64,7 @@ public class BrowserlessClient implements AutoCloseable { Map requestData = Map.of( "url", url, + "userAgent", userAgent, "options", screenshotOptions, "gotoOptions", gotoOptions ); diff --git a/code/functions/live-capture/test/nu/marginalia/livecapture/BrowserlessClientTest.java b/code/functions/live-capture/test/nu/marginalia/livecapture/BrowserlessClientTest.java index f6e0f13c..22d87061 100644 --- a/code/functions/live-capture/test/nu/marginalia/livecapture/BrowserlessClientTest.java +++ b/code/functions/live-capture/test/nu/marginalia/livecapture/BrowserlessClientTest.java @@ -1,5 +1,8 @@ package nu.marginalia.livecapture; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import nu.marginalia.WmsaHome; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -8,19 +11,74 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.junit.jupiter.Testcontainers; import org.testcontainers.utility.DockerImageName; +import java.io.IOException; +import java.net.SocketException; import java.net.URI; -import java.util.Map; +import java.util.Map;import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static java.net.NetworkInterface.getNetworkInterfaces; @Testcontainers @Tag("slow") public class BrowserlessClientTest { static GenericContainer container = new GenericContainer<>(DockerImageName.parse("browserless/chrome")) .withEnv(Map.of("TOKEN", "BROWSERLESS_TOKEN")) + .withNetworkMode("bridge") .withExposedPorts(3000); + static WireMockServer wireMockServer = new WireMockServer(WireMockConfiguration.wireMockConfig().port(18089)); + + static String localIp; @BeforeAll - public static void setup() { + public static void setup() throws IOException { container.start(); + + wireMockServer.start(); + wireMockServer.stubFor(get("/").willReturn(aResponse().withStatus(200).withBody("Ok"))); + + localIp = findLocalIp(); + } + + private static String findLocalIp() throws SocketException { + var interfaces = getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + var iface = interfaces.nextElement(); + if (iface.isLoopback()) + continue; + else if (iface.isVirtual()) + continue; + + var addresses = iface.getInetAddresses(); + + while (addresses.hasMoreElements()) { + var address = addresses.nextElement(); + + if (!address.isSiteLocalAddress()) continue; + + return address.getHostAddress(); + } + } + + return "127.0.0.1"; + } + + @Tag("flaky") + @Test + public void testInspectContentUA__Flaky() throws Exception { + try (var client = new BrowserlessClient(URI.create("http://" + container.getHost() + ":" + container.getMappedPort(3000)))) { + client.content("http://" + localIp + ":18089/", BrowserlessClient.GotoOptions.defaultValues()); + } + + wireMockServer.verify(getRequestedFor(urlEqualTo("/")).withHeader("User-Agent", equalTo(WmsaHome.getUserAgent().uaString()))); + } + + @Tag("flaky") + @Test + public void testInspectScreenshotUA__Flaky() throws Exception { + try (var client = new BrowserlessClient(URI.create("http://" + container.getHost() + ":" + container.getMappedPort(3000)))) { + client.screenshot("http://" + localIp + ":18089/", BrowserlessClient.GotoOptions.defaultValues(), BrowserlessClient.ScreenshotOptions.defaultValues()); + } + + wireMockServer.verify(getRequestedFor(urlEqualTo("/")).withHeader("User-Agent", equalTo(WmsaHome.getUserAgent().uaString()))); } @Test diff --git a/settings.gradle b/settings.gradle index 6930fd05..4269062a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -239,6 +239,7 @@ dependencyResolutionManagement { library('jooby-jte','io.jooby','jooby-jte').version(joobyVersion) library('jooby-apt','io.jooby','jooby-apt').version(joobyVersion) + library('wiremock', 'org.wiremock','wiremock').version('3.11.0') library('jte','gg.jte','jte').version('3.1.15') bundle('jetty', ['jetty-server', 'jetty-util', 'jetty-servlet'])