Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(loader) Optimize INSERT statements

Browse Source 
INSERT IGNORE is too slow.
This commit is contained in:
Viktor Lofgren 2023-11-04 17:43:55 +01:00
parent 72afa0341f
commit 7cb92195d1
1 changed files with 25 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
code/processes/loading-process/src/main/java/nu/marginalia/loading/domains/DomainLoaderService.java
View File

@ -45,24 +45,42 @@ public class DomainLoaderService {
DomainIdRegistry ret = new DomainIdRegistry(); DomainIdRegistry ret = new DomainIdRegistry();
try (var conn = dataSource.getConnection(); try (var conn = dataSource.getConnection();
var existingDomainNamesQuery = conn.prepareStatement("""
SELECT DOMAIN_NAME FROM EC_DOMAIN
""");
var selectStmt = conn.prepareStatement(""" var selectStmt = conn.prepareStatement("""
SELECT ID, DOMAIN_NAME FROM EC_DOMAIN WHERE DOMAIN_NAME=? SELECT ID, DOMAIN_NAME FROM EC_DOMAIN WHERE DOMAIN_NAME=?
""") """)
) { ) {
Set<EdgeDomain> existingDomainNames = new HashSet<>();
var existingDomainNamesRs = existingDomainNamesQuery.executeQuery();
while (existingDomainNamesRs.next()) {
existingDomainNames.add(new EdgeDomain(existingDomainNamesRs.getString(1)));
}
try (var inserter = new DomainInserter(conn, nodeId)) { try (var inserter = new DomainInserter(conn, nodeId)) {
for (var domain : readSetDomainNames(inputData)) { for (var domainName : readSetDomainNames(inputData)) {
inserter.accept(new EdgeDomain(domain)); var domain = new EdgeDomain(domainName);
domainNamesAll.add(domain); if (existingDomainNames.contains(domain))
continue;
inserter.accept(domain);
domainNamesAll.add(domainName);
} }
} }
try (var inserter = new DomainInserter(conn, -1)) { try (var inserter = new DomainInserter(conn, -1)) {
for (var domain : readReferencedDomainNames(inputData)) { for (var domainName : readReferencedDomainNames(inputData)) {
inserter.accept(new EdgeDomain(domain)); var domain = new EdgeDomain(domainName);
domainNamesAll.add(domain); if (existingDomainNames.contains(domain))
continue;
inserter.accept(domain);
domainNamesAll.add(domainName);
} }
} }
existingDomainNames.clear();
try (var updater = new DomainAffinityUpdater(conn, nodeId)) { try (var updater = new DomainAffinityUpdater(conn, nodeId)) {
for (var domain : readSetDomainNames(inputData)) { for (var domain : readSetDomainNames(inputData)) {
updater.accept(new EdgeDomain(domain)); updater.accept(new EdgeDomain(domain));
@ -164,6 +182,7 @@ public class DomainLoaderService {
statement.close(); statement.close();
} }
} }
private static class DomainAffinityUpdater implements AutoCloseable { private static class DomainAffinityUpdater implements AutoCloseable {
private final PreparedStatement statement; private final PreparedStatement statement;
private final int nodeAffinity; private final int nodeAffinity;