Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(loader) Tidy up code

Browse Source
This commit is contained in:
Viktor Lofgren 2024-07-28 21:36:42 +02:00
parent 7d51cf882f
commit afe56c7cf1
1 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
code/processes/loading-process/java/nu/marginalia/loading/domains/DomainLoaderService.java
View File

@ -43,7 +43,8 @@ public class DomainLoaderService {
FETCH_ALL,
DONE
}
/** Read the domain names from each parquet file
/** Read the domain names from each input file
* compare with SQL domain database, fetch those
* that exist, insert those that don't.
*/
@ -54,11 +55,8 @@ public class DomainLoaderService {
DomainIdRegistry ret = new DomainIdRegistry();
try (var conn = dataSource.getConnection();
var taskHeartbeat = heartbeat.createProcessTaskHeartbeat(Steps.class, "DOMAIN_IDS");
var selectStmt = conn.prepareStatement("""
SELECT ID, LOWER(DOMAIN_NAME) FROM EC_DOMAIN
""")
) {
var taskHeartbeat = heartbeat.createProcessTaskHeartbeat(Steps.class, "DOMAIN_IDS"))
{
taskHeartbeat.progress(Steps.PREP_DATA);
Collection<SlopPageRef<SlopDomainRecord>> domainPageRefs = inputData.listDomainPages();
@ -128,14 +126,19 @@ public class DomainLoaderService {
}
taskHeartbeat.progress(Steps.FETCH_ALL);
selectStmt.setFetchSize(1000);
var rs = selectStmt.executeQuery();
while (rs.next()) {
String domain = rs.getString(2);
// Fetch the ID for all domains that we have information about
try (var selectStmt = conn.prepareStatement("SELECT ID, LOWER(DOMAIN_NAME) FROM EC_DOMAIN")) {
if (domainNamesAll.contains(domain)) {
ret.add(domain, rs.getInt(1));
selectStmt.setFetchSize(1000);
var rs = selectStmt.executeQuery();
while (rs.next()) {
String domain = rs.getString(2);
if (domainNamesAll.contains(domain)) {
ret.add(domain, rs.getInt(1));
}
}
}