From affcf8cf414aadb24a0533ca2fee013c0651d4ae Mon Sep 17 00:00:00 2001
From: Viktor Lofgren <vlofgren@marginalia.nu>
Date: Sun, 2 Apr 2023 09:43:43 +0200
Subject: [PATCH] Load test tool

---
 code/tools/load-test/build.gradle             | 33 ++++++++
 code/tools/load-test/readme.md                |  9 +++
 .../nu/marginalia/load_test/LoadTestMain.java | 76 +++++++++++++++++++
 settings.gradle                               |  1 +
 4 files changed, 119 insertions(+)
 create mode 100644 code/tools/load-test/build.gradle
 create mode 100644 code/tools/load-test/readme.md
 create mode 100644 code/tools/load-test/src/main/java/nu/marginalia/load_test/LoadTestMain.java

diff --git a/code/tools/load-test/build.gradle b/code/tools/load-test/build.gradle
new file mode 100644
index 00000000..f0d9d84b
--- /dev/null
+++ b/code/tools/load-test/build.gradle
@@ -0,0 +1,33 @@
+plugins {
+    id 'java'
+    id "io.freefair.lombok" version "5.3.3.3"
+
+    id 'jvm-test-suite'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(17))
+    }
+}
+
+dependencies {
+    implementation project(':code:common:config')
+    implementation project(':code:common:model')
+    implementation project(':code:libraries:term-frequency-dict')
+    implementation libs.lombok
+    annotationProcessor libs.lombok
+    implementation libs.bundles.slf4j
+    implementation libs.notnull
+}
+
+
+test {
+    useJUnitPlatform()
+}
+
+task fastTests(type: Test) {
+    useJUnitPlatform {
+        excludeTags "slow"
+    }
+}
diff --git a/code/tools/load-test/readme.md b/code/tools/load-test/readme.md
new file mode 100644
index 00000000..3af49738
--- /dev/null
+++ b/code/tools/load-test/readme.md
@@ -0,0 +1,9 @@
+# Load Test
+
+Performs random queries to puts load on the local 
+Marginalia deployment to enable profiling of the index. 
+
+Run in the IDE.
+
+Configure the profiler to look at port 7021 for the index service, 
+and 7023 for the search service.
\ No newline at end of file
diff --git a/code/tools/load-test/src/main/java/nu/marginalia/load_test/LoadTestMain.java b/code/tools/load-test/src/main/java/nu/marginalia/load_test/LoadTestMain.java
new file mode 100644
index 00000000..16c2d3be
--- /dev/null
+++ b/code/tools/load-test/src/main/java/nu/marginalia/load_test/LoadTestMain.java
@@ -0,0 +1,76 @@
+package nu.marginalia.load_test;
+
+import nu.marginalia.WmsaHome;
+import nu.marginalia.term_frequency_dict.TermFrequencyDict;
+import org.apache.logging.log4j.util.Strings;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class LoadTestMain {
+    private static List<String> commonWords;
+    public static void main(String... args) throws URISyntaxException, IOException, InterruptedException {
+        commonWords = loadCommonWords();
+
+
+        System.out.println(commonWords.size());
+
+        HttpClient client = HttpClient.newHttpClient();
+
+        List<Long> times = new ArrayList<>();
+
+        for (int i = 0; i < 10000; i++) {
+            String uri = "http://127.0.0.1:8080/search?query=%s&profile=corpo".formatted(
+                    Strings.join(pickNCommonWords(2), '+')
+            );
+
+            HttpRequest req = HttpRequest.newBuilder(new URI(uri))
+                    .build();
+
+
+            long startTime = System.currentTimeMillis();
+            var rsp = client.send(req, HttpResponse.BodyHandlers.ofString());
+            long stopTime = System.currentTimeMillis();
+
+            times.add(stopTime - startTime);
+            if (times.size() > 100) {
+                System.out.println(times.stream().mapToLong(Long::longValue).average().orElse(-1));
+                times.clear();
+            }
+//            System.out.println(stopTime - startTime);
+        }
+    }
+
+    private static List<String> loadCommonWords() {
+        var dict = new TermFrequencyDict(WmsaHome.getLanguageModels());
+
+
+        try (var lines = Files.lines(Path.of("/usr/share/dict/american-english"))) {
+            return lines.map(String::toLowerCase).filter(term -> dict.getTermFreq(term) > 100000).collect(Collectors.toList());
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+    }
+
+    static List<String> pickNCommonWords(int n) {
+        assert commonWords.size() > 10*n;
+
+        Set<String> words = new HashSet<>(n);
+        Random r = new Random(System.nanoTime());
+        while (words.size() < n) {
+            words.add(commonWords.get(r.nextInt(0, commonWords.size())));
+        }
+
+        return new ArrayList<>(words);
+    }
+}
diff --git a/settings.gradle b/settings.gradle
index a0908d99..0868e447 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -63,6 +63,7 @@ include 'code:process-models:crawling-model'
 include 'code:tools:term-frequency-extractor'
 include 'code:tools:crawl-job-extractor'
 include 'code:tools:experiment-runner'
+include 'code:tools:load-test'
 
 include 'third-party:porterstemmer'
 include 'third-party:xz'