Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Tidy up LoaderMain a bit

Browse Source
This commit is contained in:
vlofgren 2022-07-18 17:22:22 +02:00
parent 3d1031f8e4
commit 15bd54ef9f
1 changed files with 9 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
marginalia_nu/src/main/java/nu/marginalia/wmsa/edge/converting/LoaderMain.java
View File

@ -3,7 +3,6 @@ package nu.marginalia.wmsa.edge.converting;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.google.inject.Injector; import com.google.inject.Injector;
import com.zaxxer.hikari.HikariDataSource;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import nu.marginalia.wmsa.configuration.module.DatabaseModule; import nu.marginalia.wmsa.configuration.module.DatabaseModule;
import nu.marginalia.wmsa.edge.converting.interpreter.Instruction; import nu.marginalia.wmsa.edge.converting.interpreter.Instruction;
@ -25,14 +24,15 @@ import java.util.concurrent.atomic.AtomicInteger;
public class LoaderMain { public class LoaderMain {
private static final Logger logger = LoggerFactory.getLogger(LoaderMain.class);
private final Path processDir; private final Path processDir;
private final EdgeCrawlPlan plan; private final EdgeCrawlPlan plan;
private final ConvertedDomainReader instructionsReader; private final ConvertedDomainReader instructionsReader;
private final HikariDataSource dataSource;
private static final Logger logger = LoggerFactory.getLogger(LoaderMain.class);
private final LoaderFactory loaderFactory; private final LoaderFactory loaderFactory;
private final EdgeIndexClient indexClient; private final EdgeIndexClient indexClient;
private volatile boolean running = true; private volatile boolean running = true;
final Thread processorThread = new Thread(this::processor, "Processor Thread"); final Thread processorThread = new Thread(this::processor, "Processor Thread");
@ -56,14 +56,12 @@ public class LoaderMain {
@Inject @Inject
public LoaderMain(EdgeCrawlPlan plan, public LoaderMain(EdgeCrawlPlan plan,
ConvertedDomainReader instructionsReader, ConvertedDomainReader instructionsReader,
HikariDataSource dataSource,
LoaderFactory loaderFactory, LoaderFactory loaderFactory,
EdgeIndexClient indexClient) { EdgeIndexClient indexClient) {
this.processDir = plan.process.getDir(); this.processDir = plan.process.getDir();
this.plan = plan; this.plan = plan;
this.instructionsReader = instructionsReader; this.instructionsReader = instructionsReader;
this.dataSource = dataSource;
this.loaderFactory = loaderFactory; this.loaderFactory = loaderFactory;
this.indexClient = indexClient; this.indexClient = indexClient;
@ -79,7 +77,7 @@ public class LoaderMain {
LoaderMain.loadTotal = loadTotal.get(); LoaderMain.loadTotal = loadTotal.get();
WorkLog.readLog(logFile, entry -> { WorkLog.readLog(logFile, entry -> {
load(entry.path(), entry.cnt()); load(plan, entry.path(), entry.cnt());
}); });
running = false; running = false;
@ -90,15 +88,9 @@ public class LoaderMain {
} }
private volatile static int loadTotal; private volatile static int loadTotal;
private static final int loaded = 0;
private void load(String path, int cnt) {
String first = path.substring(0, 2);
String second = path.substring(2, 4);
Path destDir = processDir.resolve(first).resolve(second).resolve(path);
private void load(EdgeCrawlPlan plan, String path, int cnt) {
Path destDir = plan.getProcessedFilePath(path);
try { try {
var loader = loaderFactory.create(cnt); var loader = loaderFactory.create(cnt);
var instructions = instructionsReader.read(destDir, cnt); var instructions = instructionsReader.read(destDir, cnt);
@ -120,7 +112,8 @@ public class LoaderMain {
loader.finish(); loader.finish();
long loadTime = System.currentTimeMillis() - startTime; long loadTime = System.currentTimeMillis() - startTime;
taskStats.observe(loadTime); taskStats.observe(loadTime);
logger.info("Loaded {}/{} : {} ({}) {}ms {} l/s", taskStats.getCount(), loadTotal, path, loader.data.sizeHint, loadTime, taskStats.avgTime()); logger.info("Loaded {}/{} : {} ({}) {}ms {} l/s", taskStats.getCount(),
loadTotal, path, loader.data.sizeHint, loadTime, taskStats.avgTime());
} }
} }