(test) Clean up test usage of migrations

Several tests were manually running migrations in a large copy-paste blob of code.  This makes the test less useful as it's possible to break the code while keeping the tests green by introducing a new migration that never gets run in the tests, and it's also difficult to reason about what the tests are doing.

A new test helper library is introduced with a TestMigrationLoader that can both run Flyway migrations, or load specific migrations in the cases a specific set of migrations need to be loaded.   Existing tests are migrated to use the new code.

code/common/config/build.gradle

@@ -27,9 +27,11 @@ dependencies {
     testImplementation libs.bundles.junit
 
 
+    testImplementation project(':code:libraries:test-helpers')
 
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 
 }

code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java

@@ -20,7 +20,7 @@ public class NodeConfigurationService {
     public NodeConfiguration create(int id, String description, boolean acceptQueries, boolean keepWarcs) throws SQLException {
         try (var conn = dataSource.getConnection();
              var is = conn.prepareStatement("""
-                     INSERT IGNORE INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?, ?, ?)
+                     INSERT IGNORE INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES, KEEP_WARCS) VALUES(?, ?, ?, ?)
                      """)
         )
         {

code/common/config/src/test/java/nu/marginalia/nodecfg/NodeConfigurationServiceTest.java

@@ -3,6 +3,7 @@ package nu.marginalia.nodecfg;
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.storage.FileStorageService;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -30,7 +31,6 @@ public class NodeConfigurationServiceTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_11_0_003__node_configuration.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -45,6 +45,8 @@ public class NodeConfigurationServiceTest {
 
         dataSource = new HikariDataSource(config);
 
+        TestMigrationLoader.flywayMigration(dataSource);
+
         nodeConfigurationService = new NodeConfigurationService(dataSource);
     }
 

code/common/config/src/test/java/nu/marginalia/storage/FileStorageServiceTest.java

@@ -5,6 +5,7 @@ import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.storage.model.FileStorageBaseType;
 import nu.marginalia.storage.model.FileStorageType;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.ExecutionMode;
@@ -33,7 +34,6 @@ public class FileStorageServiceTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_07_0_004__file_storage.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -50,32 +50,7 @@ public class FileStorageServiceTest {
 
         dataSource = new HikariDataSource(config);
 
-        // apply migrations
+        TestMigrationLoader.flywayMigration(dataSource);
-
-        List<String> migrations = List.of(
-                "db/migration/V23_11_0_000__file_storage_node.sql",
-                "db/migration/V23_11_0_002__file_storage_state.sql",
-                "db/migration/V23_11_0_004__file_storage_base_type.sql"
-                );
-        for (String migration : migrations) {
-            try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
-                    "Could not load migration script " + migration);
-                 var conn = dataSource.getConnection();
-                 var stmt = conn.createStatement()
-            ) {
-                String script = new String(resource.readAllBytes());
-                String[] cmds = script.split("\\s*;\\s*");
-                for (String cmd : cmds) {
-                    if (cmd.isBlank())
-                        continue;
-                    System.out.println(cmd);
-                    stmt.executeUpdate(cmd);
-                }
-            }
-            catch (IOException|SQLException ex) {
-
-            }
-        }
     }
 
 

code/common/db/build.gradle

@@ -52,6 +52,7 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 }
 
 flyway {

code/common/db/src/test/java/nu/marginalia/db/DomainTypesTest.java

@@ -3,6 +3,7 @@ package nu.marginalia.db;
 import com.google.common.collect.Sets;
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -26,7 +27,6 @@ public class DomainTypesTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_07_0_001__domain_type.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -40,6 +40,7 @@ public class DomainTypesTest {
         config.setPassword("wmsa");
 
         dataSource = new HikariDataSource(config);
+        TestMigrationLoader.flywayMigration(dataSource);
 
         domainTypes = new DomainTypes(dataSource);
     }

code/common/linkdb/build.gradle

@@ -38,5 +38,6 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 }
 

code/libraries/message-queue/build.gradle

@@ -29,4 +29,5 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 }

code/libraries/message-queue/src/test/java/nu/marginalia/actor/ActorRecordMachineTest.java

@@ -11,6 +11,7 @@ import nu.marginalia.actor.state.Resume;
 import nu.marginalia.mq.MessageQueueFactory;
 import nu.marginalia.mq.MqTestUtil;
 import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.parallel.Execution;
 import org.testcontainers.containers.MariaDBContainer;
@@ -31,7 +32,6 @@ public class ActorRecordMachineTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -51,6 +51,8 @@ public class ActorRecordMachineTest {
         config.setPassword("wmsa");
 
         dataSource = new HikariDataSource(config);
+        TestMigrationLoader.flywayMigration(dataSource);
+
         persistence = new MqPersistence(dataSource);
         messageQueueFactory = new MessageQueueFactory(persistence);
     }

code/libraries/message-queue/src/test/java/nu/marginalia/mq/outbox/MqOutboxTest.java

@@ -7,6 +7,7 @@ import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.MqTestUtil;
 import nu.marginalia.mq.inbox.*;
 import nu.marginalia.mq.persistence.MqPersistence;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.*;
 import org.testcontainers.containers.MariaDBContainer;
 import org.testcontainers.junit.jupiter.Container;
@@ -27,7 +28,6 @@ public class MqOutboxTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -45,6 +45,7 @@ public class MqOutboxTest {
         config.setPassword("wmsa");
 
         dataSource = new HikariDataSource(config);
+        TestMigrationLoader.flywayMigration(dataSource);
     }
 
     @AfterAll

code/libraries/message-queue/src/test/java/nu/marginalia/mq/persistence/MqPersistenceTest.java

@@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.mq.MqMessageState;
 import nu.marginalia.mq.MqTestUtil;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.*;
 import org.testcontainers.containers.MariaDBContainer;
 import org.testcontainers.junit.jupiter.Container;
@@ -24,7 +25,6 @@ public class MqPersistenceTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_07_0_003__message_queue.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -47,6 +47,7 @@ public class MqPersistenceTest {
 
         dataSource = new HikariDataSource(config);
         persistence = new MqPersistence(dataSource);
+        TestMigrationLoader.flywayMigration(dataSource);
     }
 
     @AfterAll

code/libraries/test-helpers/build.gradle

@@ -0,0 +1,22 @@
+plugins {
+    id 'java'
+}
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
+}
+
+dependencies {
+    implementation libs.bundles.slf4j
+    implementation libs.bundles.mariadb
+    implementation libs.bundles.slf4j.test
+    implementation libs.bundles.junit
+    implementation libs.mockito
+    implementation libs.bundles.flyway
+}
+
+test {
+    useJUnitPlatform()
+}

code/libraries/test-helpers/src/main/java/nu/marginalia/test/TestMigrationLoader.java

@@ -0,0 +1,45 @@
+package nu.marginalia.test;
+
+import com.zaxxer.hikari.HikariDataSource;
+import org.flywaydb.core.Flyway;
+import org.junit.jupiter.api.Assertions;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Objects;
+
+public class TestMigrationLoader {
+
+    /** Use flyway to load migrations from the classpath and run them against the database. */
+    public static void flywayMigration(HikariDataSource dataSource) {
+        Flyway.configure().dataSource(dataSource).load().migrate();
+    }
+
+    /** Load specific migrations from the classpath and run them against the database. */
+    public static void loadMigrations(HikariDataSource dataSource, String... migrations) {
+        for (var migration : migrations) {
+            loadMigration(dataSource, migration);
+        }
+    }
+
+    /** Load specific migration from the classpath and run them against the database. */
+    public static void loadMigration(HikariDataSource dataSource, String migration) {
+        try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
+                "Could not load migration script " + migration);
+             var conn = dataSource.getConnection();
+             var stmt = conn.createStatement()
+        ) {
+            String script = new String(resource.readAllBytes());
+            String[] cmds = script.split("\\s*;\\s*");
+            for (String cmd : cmds) {
+                if (cmd.isBlank())
+                    continue;
+                System.out.println(cmd);
+                stmt.executeUpdate(cmd);
+            }
+        }
+        catch (IOException | SQLException ex) {
+            Assertions.fail("Failed to load migration " + migration, ex);
+        }
+    }
+}

code/processes/loading-process/build.gradle

@@ -56,5 +56,6 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 }
 

code/processes/loading-process/src/test/java/nu/marginalia/loading/domains/DomainLoaderServiceTest.java

@@ -11,6 +11,7 @@ import nu.marginalia.model.processed.DomainLinkRecord;
 import nu.marginalia.model.processed.DomainRecord;
 import nu.marginalia.process.control.ProcessAdHocTaskHeartbeat;
 import nu.marginalia.process.control.ProcessHeartbeat;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.*;
 import org.mockito.Mockito;
 import org.testcontainers.containers.MariaDBContainer;
@@ -31,14 +32,6 @@ class DomainLoaderServiceTest {
     List<Path> toDelete = new ArrayList<>();
     ProcessHeartbeat heartbeat;
 
-    @Container
-    static MariaDBContainer<?> mariaDBContainer = new MariaDBContainer<>("mariadb")
-            .withDatabaseName("WMSA_prod")
-            .withUsername("wmsa")
-            .withPassword("wmsa")
-            .withInitScript("db/migration/V23_06_0_000__base.sql")
-            .withNetworkAliases("mariadb");
-
     @BeforeEach
     public void setUp() {
         heartbeat = Mockito.mock(ProcessHeartbeat.class);

code/processes/website-adjacencies-calculator/build.gradle

@@ -40,4 +40,5 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 }

code/services-application/api-service/build.gradle

@@ -51,5 +51,6 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 }
 

code/services-application/api-service/src/test/java/nu/marginalia/api/svc/LicenseServiceTest.java

@@ -3,6 +3,7 @@ package nu.marginalia.api.svc;
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.*;
 import org.testcontainers.containers.MariaDBContainer;
 import org.testcontainers.junit.jupiter.Container;
@@ -21,7 +22,6 @@ class LicenseServiceTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_06_0_006__api_key.sql")
             .withNetworkAliases("mariadb");
 
     private static LicenseService service;
@@ -32,6 +32,7 @@ class LicenseServiceTest {
         mariaDBContainer.start();
 
         dataSource = getConnection(mariaDBContainer.getJdbcUrl());
+        TestMigrationLoader.flywayMigration(dataSource);
 
         try (var conn = dataSource.getConnection();
              var stmt = conn.prepareStatement("""

code/services-core/control-service/build.gradle

@@ -63,6 +63,7 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
+    testImplementation project(':code:libraries:test-helpers')
 }
 
 

code/services-core/control-service/src/test/java/nu/marginalia/control/svc/ApiKeyServiceTest.java

@@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.control.app.model.ApiKeyModel;
 import nu.marginalia.control.app.svc.ApiKeyService;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.parallel.Execution;
 import org.testcontainers.containers.MariaDBContainer;
@@ -25,7 +26,6 @@ public class ApiKeyServiceTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_06_0_006__api_key.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -37,6 +37,7 @@ public class ApiKeyServiceTest {
         config.setPassword("wmsa");
 
         dataSource = new HikariDataSource(config);
+        TestMigrationLoader.flywayMigration(dataSource);
     }
 
     @AfterAll

code/services-core/control-service/src/test/java/nu/marginalia/control/svc/HeartbeatServiceTest.java

@@ -5,6 +5,7 @@ import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.control.sys.model.TaskHeartbeat;
 import nu.marginalia.control.sys.svc.HeartbeatService;
 import nu.marginalia.service.control.ServiceEventLog;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -32,10 +33,8 @@ class HeartbeatServiceTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_07_0_007__task_status.sql")
             .withNetworkAliases("mariadb");
 
-
     static HikariDataSource dataSource;
     @BeforeAll
     public static void setup() {
@@ -46,25 +45,7 @@ class HeartbeatServiceTest {
 
         dataSource = new HikariDataSource(config);
 
-        List<String> migrations = List.of("db/migration/V23_11_0_001__heartbeat_node.sql");
+        TestMigrationLoader.flywayMigration(dataSource);
-        for (String migration : migrations) {
-            try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
-                    "Could not load migration script " + migration);
-                 var conn = dataSource.getConnection();
-                 var stmt = conn.createStatement()
-            ) {
-                String script = new String(resource.readAllBytes());
-                String[] cmds = script.split("\\s*;\\s*");
-                for (String cmd : cmds) {
-                    if (cmd.isBlank())
-                        continue;
-                    System.out.println(cmd);
-                    stmt.executeUpdate(cmd);
-                }
-            } catch (IOException | SQLException ex) {
-
-            }
-        }
     }
 
     @AfterAll

code/services-core/executor-service/build.gradle

@@ -68,6 +68,6 @@ dependencies {
     testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4')
     testImplementation 'org.testcontainers:mariadb:1.17.4'
     testImplementation 'org.testcontainers:junit-jupiter:1.17.4'
-
+    testImplementation project(':code:libraries:test-helpers')
 
 }

code/services-core/executor-service/src/test/java/nu/marginalia/svc/DomainListRefreshServiceTest.java

@@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 import nu.marginalia.db.DomainTypes;
 import nu.marginalia.service.module.ServiceConfiguration;
+import nu.marginalia.test.TestMigrationLoader;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -29,7 +30,6 @@ class DomainListRefreshServiceTest {
             .withDatabaseName("WMSA_prod")
             .withUsername("wmsa")
             .withPassword("wmsa")
-            .withInitScript("db/migration/V23_06_0_000__base.sql")
             .withNetworkAliases("mariadb");
 
     static HikariDataSource dataSource;
@@ -43,31 +43,7 @@ class DomainListRefreshServiceTest {
 
         dataSource = new HikariDataSource(config);
 
-        // apply migrations
+        TestMigrationLoader.flywayMigration(dataSource);
-
-        List<String> migrations = List.of(
-                "db/migration/V23_06_0_003__crawl-queue.sql",
-                "db/migration/V23_07_0_001__domain_type.sql",
-                "db/migration/V23_11_0_007__domain_node_affinity.sql"
-        );
-        for (String migration : migrations) {
-            try (var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(migration),
-                    "Could not load migration script " + migration);
-                 var conn = dataSource.getConnection();
-                 var stmt = conn.createStatement()
-            ) {
-                String script = new String(resource.readAllBytes());
-                String[] cmds = script.split("\\s*;\\s*");
-                for (String cmd : cmds) {
-                    if (cmd.isBlank())
-                        continue;
-                    System.out.println(cmd);
-                    stmt.executeUpdate(cmd);
-                }
-            } catch (IOException | SQLException ex) {
-
-            }
-        }
     }
 
     @AfterAll

settings.gradle

@@ -23,6 +23,7 @@ include 'code:libraries:blocking-thread-pool'
 include 'code:libraries:braille-block-punch-cards'
 include 'code:libraries:language-processing'
 include 'code:libraries:term-frequency-dict'
+include 'code:libraries:test-helpers'
 
 include 'code:libraries:message-queue'