Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Make the crawler report crawling progress correctly when stopped and resumed.

Browse Source
This commit is contained in:
Viktor Lofgren 2024-09-26 18:30:29 +02:00
parent 336d6fdd14
commit 4565bfe359
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code/processes/crawling-process/java/nu/marginalia/crawl/CrawlerMain.java
View File

@ -180,6 +180,11 @@ public class CrawlerMain extends ProcessMainClass {
WarcArchiverIf warcArchiver = warcArchiverFactory.get(outputDir); WarcArchiverIf warcArchiver = warcArchiverFactory.get(outputDir);
AnchorTagsSource anchorTagsSource = anchorTagsSourceFactory.create(specProvider.getDomains()) AnchorTagsSource anchorTagsSource = anchorTagsSourceFactory.create(specProvider.getDomains())
) { ) {
// Set the number of tasks done to the number of tasks that are already finished,
// (this happens when the process is restarted after a crash or a shutdown)
tasksDone.set(workLog.countFinishedJobs());
// Process the crawl tasks
try (var specStream = specProvider.stream()) { try (var specStream = specProvider.stream()) {
specStream specStream
.takeWhile((e) -> abortMonitor.isAlive()) .takeWhile((e) -> abortMonitor.isAlive())