Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 13:09:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Tidy up LoaderMain a bit

Browse Source
This commit is contained in:
vlofgren 2022-07-18 17:22:22 +02:00
parent 3d1031f8e4
commit 15bd54ef9f
1 changed files with 9 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/LoaderMain.java
View File

@ -3,7 +3,6 @@ package nu.marginalia.wmsa.edge.converting;
import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.zaxxer.hikari.HikariDataSource;
import lombok.SneakyThrows;
import nu.marginalia.wmsa.configuration.module.DatabaseModule;
import nu.marginalia.wmsa.edge.converting.interpreter.Instruction;
@ -25,14 +24,15 @@ import java.util.concurrent.atomic.AtomicInteger;
public class LoaderMain {
private static final Logger logger = LoggerFactory.getLogger(LoaderMain.class);
private final Path processDir;
private final EdgeCrawlPlan plan;
private final ConvertedDomainReader instructionsReader;
private final HikariDataSource dataSource;
private static final Logger logger = LoggerFactory.getLogger(LoaderMain.class);
private final LoaderFactory loaderFactory;
private final EdgeIndexClient indexClient;
private volatile boolean running = true;
final Thread processorThread = new Thread(this::processor, "Processor Thread");
@ -56,14 +56,12 @@ public class LoaderMain {
@Inject
public LoaderMain(EdgeCrawlPlan plan,
ConvertedDomainReader instructionsReader,
HikariDataSource dataSource,
LoaderFactory loaderFactory,
EdgeIndexClient indexClient) {
this.processDir = plan.process.getDir();
this.plan = plan;
this.instructionsReader = instructionsReader;
this.dataSource = dataSource;
this.loaderFactory = loaderFactory;
this.indexClient = indexClient;
@ -79,7 +77,7 @@ public class LoaderMain {
LoaderMain.loadTotal = loadTotal.get();
WorkLog.readLog(logFile, entry -> {
load(entry.path(), entry.cnt());
load(plan, entry.path(), entry.cnt());
});
running = false;
@ -90,15 +88,9 @@ public class LoaderMain {
}
private volatile static int loadTotal;
private static final int loaded = 0;
private void load(String path, int cnt) {
String first = path.substring(0, 2);
String second = path.substring(2, 4);
Path destDir = processDir.resolve(first).resolve(second).resolve(path);
private void load(EdgeCrawlPlan plan, String path, int cnt) {
Path destDir = plan.getProcessedFilePath(path);
try {
var loader = loaderFactory.create(cnt);
var instructions = instructionsReader.read(destDir, cnt);
@ -120,7 +112,8 @@ public class LoaderMain {
loader.finish();
long loadTime = System.currentTimeMillis() - startTime;
taskStats.observe(loadTime);
logger.info("Loaded {}/{} : {} ({}) {}ms {} l/s", taskStats.getCount(), loadTotal, path, loader.data.sizeHint, loadTime, taskStats.avgTime());
logger.info("Loaded {}/{} : {} ({}) {}ms {} l/s", taskStats.getCount(),
loadTotal, path, loader.data.sizeHint, loadTime, taskStats.avgTime());
}
}