diff --git a/code/common/db/src/main/resources/db/migration/V23_11_0_008__purge_procedure.sql b/code/common/db/src/main/resources/db/migration/V23_11_0_008__purge_procedure.sql new file mode 100644 index 00000000..ac0df87b --- /dev/null +++ b/code/common/db/src/main/resources/db/migration/V23_11_0_008__purge_procedure.sql @@ -0,0 +1,10 @@ +ALTER TABLE WMSA_prod.EC_DOMAIN_LINK +MODIFY COLUMN ID BIGINT NOT NULL AUTO_INCREMENT; + +CREATE OR REPLACE PROCEDURE PURGE_LINKS_TABLE (IN nodeId INT) +BEGIN + DELETE EC_DOMAIN_LINK + FROM EC_DOMAIN_LINK INNER JOIN WMSA_prod.EC_DOMAIN + ON EC_DOMAIN_LINK.SOURCE_DOMAIN_ID = EC_DOMAIN.ID + WHERE NODE_AFFINITY = nodeId; +END; diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/links/DomainLinksLoaderService.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/links/DomainLinksLoaderService.java index 78e254e6..6f3a6d8f 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/links/DomainLinksLoaderService.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/links/DomainLinksLoaderService.java @@ -3,6 +3,7 @@ package nu.marginalia.loading.links; import com.google.inject.Inject; import com.google.inject.Singleton; import com.zaxxer.hikari.HikariDataSource; +import nu.marginalia.ProcessConfiguration; import nu.marginalia.io.processed.DomainLinkRecordParquetFileReader; import nu.marginalia.io.processed.ProcessedDataFileNames; import nu.marginalia.loading.LoaderInputData; @@ -23,10 +24,12 @@ public class DomainLinksLoaderService { private final HikariDataSource dataSource; private static final Logger logger = LoggerFactory.getLogger(DomainLinksLoaderService.class); - + private final int nodeId; @Inject - public DomainLinksLoaderService(HikariDataSource dataSource) { + public DomainLinksLoaderService(HikariDataSource dataSource, + ProcessConfiguration processConfiguration) { this.dataSource = dataSource; + this.nodeId = processConfiguration.node(); } public boolean loadLinks(DomainIdRegistry domainIdRegistry, @@ -54,11 +57,12 @@ public class DomainLinksLoaderService { } private void dropLinkData() throws SQLException { - logger.info("Truncating EC_DOMAIN_LINK"); + logger.info("Clearing EC_DOMAIN_LINK"); try (var conn = dataSource.getConnection(); - var stmt = conn.createStatement()) { - stmt.executeUpdate("TRUNCATE TABLE EC_DOMAIN_LINK"); + var call = conn.prepareCall("CALL PURGE_LINKS_TABLE(?)")) { + call.setInt(1, nodeId); + call.executeUpdate(); } } @@ -84,7 +88,7 @@ public class DomainLinksLoaderService { connection = dataSource.getConnection(); insertStatement = connection.prepareStatement(""" - INSERT INTO EC_DOMAIN_LINK(SOURCE_DOMAIN_ID, DEST_DOMAIN_ID) + INSERT IGNORE INTO EC_DOMAIN_LINK(SOURCE_DOMAIN_ID, DEST_DOMAIN_ID) VALUES (?, ?) """); }