diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainIdRegistry.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainIdRegistry.java index 2ab6ba46..bccc3ed3 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainIdRegistry.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainIdRegistry.java @@ -5,7 +5,7 @@ import java.util.Map; /** Maps domain names to domain ids */ public class DomainIdRegistry { - private final Map domainIds = new HashMap<>(); + private final Map domainIds = new HashMap<>(10_000); public int getDomainId(String domainName) { Integer id = domainIds.get(domainName.toLowerCase()); diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainLoaderService.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainLoaderService.java index fa34e07c..6739f8e7 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainLoaderService.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainLoaderService.java @@ -42,12 +42,12 @@ public class DomainLoaderService { public DomainIdRegistry getOrCreateDomainIds(LoaderInputData inputData) throws IOException, SQLException { - Set domainNamesAll = new HashSet<>(); + Set domainNamesAll = new HashSet<>(100_000); DomainIdRegistry ret = new DomainIdRegistry(); try (var conn = dataSource.getConnection(); var selectStmt = conn.prepareStatement(""" - SELECT ID, DOMAIN_NAME FROM EC_DOMAIN WHERE DOMAIN_NAME=? + SELECT ID FROM EC_DOMAIN WHERE DOMAIN_NAME=? """) ) { @@ -70,6 +70,7 @@ public class DomainLoaderService { } } + selectStmt.setFetchSize(1000); for (var domain : domainNamesAll) { selectStmt.setString(1, domain); var rs = selectStmt.executeQuery(); @@ -133,7 +134,7 @@ public class DomainLoaderService { return true; } - private class DomainInserter implements AutoCloseable { + private static class DomainInserter implements AutoCloseable { private final PreparedStatement statement; private final int nodeAffinity;