Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(loader) Tidy up code

Browse Source
This commit is contained in:
Viktor Lofgren 2024-07-28 21:36:42 +02:00
parent 7d51cf882f
commit afe56c7cf1
1 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
code/processes/loading-process/java/nu/marginalia/loading/domains/DomainLoaderService.java
View File

@ -43,7 +43,8 @@ public class DomainLoaderService {
FETCH_ALL, FETCH_ALL,
DONE DONE
} }
/** Read the domain names from each parquet file
/** Read the domain names from each input file
* compare with SQL domain database, fetch those * compare with SQL domain database, fetch those
* that exist, insert those that don't. * that exist, insert those that don't.
*/ */
@ -54,11 +55,8 @@ public class DomainLoaderService {
DomainIdRegistry ret = new DomainIdRegistry(); DomainIdRegistry ret = new DomainIdRegistry();
try (var conn = dataSource.getConnection(); try (var conn = dataSource.getConnection();
var taskHeartbeat = heartbeat.createProcessTaskHeartbeat(Steps.class, "DOMAIN_IDS"); var taskHeartbeat = heartbeat.createProcessTaskHeartbeat(Steps.class, "DOMAIN_IDS"))
var selectStmt = conn.prepareStatement(""" {
SELECT ID, LOWER(DOMAIN_NAME) FROM EC_DOMAIN
""")
) {
taskHeartbeat.progress(Steps.PREP_DATA); taskHeartbeat.progress(Steps.PREP_DATA);
Collection<SlopPageRef<SlopDomainRecord>> domainPageRefs = inputData.listDomainPages(); Collection<SlopPageRef<SlopDomainRecord>> domainPageRefs = inputData.listDomainPages();
@ -128,14 +126,19 @@ public class DomainLoaderService {
} }
taskHeartbeat.progress(Steps.FETCH_ALL); taskHeartbeat.progress(Steps.FETCH_ALL);
selectStmt.setFetchSize(1000);
var rs = selectStmt.executeQuery(); // Fetch the ID for all domains that we have information about
while (rs.next()) { try (var selectStmt = conn.prepareStatement("SELECT ID, LOWER(DOMAIN_NAME) FROM EC_DOMAIN")) {
String domain = rs.getString(2);
if (domainNamesAll.contains(domain)) { selectStmt.setFetchSize(1000);
ret.add(domain, rs.getInt(1));
var rs = selectStmt.executeQuery();
while (rs.next()) {
String domain = rs.getString(2);
if (domainNamesAll.contains(domain)) {
ret.add(domain, rs.getInt(1));
}
} }
} }