Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Terminate crawler after a few hours of no progress

Browse Source
This commit is contained in:
Viktor Lofgren 2023-10-26 12:49:28 +02:00
parent 0f637fb722
commit a497e4c920
2 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java
View File

@ -3,6 +3,7 @@ package nu.marginalia.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
@ -47,12 +48,15 @@ public class SimpleBlockingThreadPool {
this.shutDown = true;
}
public void shutDownNow() {
public void shutDownNow() throws InterruptedException {
this.shutDown = true;
tasks.clear();
for (Thread worker : workers) {
worker.interrupt();
}
for (Thread worker : workers) {
worker.join(Duration.ofMinutes(5));
}
}
private void worker() {

16
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java
View File

@ -151,12 +151,20 @@ public class CrawlerMain {
logger.info("Shutting down the pool, waiting for tasks to complete...");
pool.shutDown();
do {
System.out.println("Waiting for pool to terminate... " + pool.getActiveCount() + " remaining");
} while (!pool.awaitTermination(60, TimeUnit.SECONDS));
int activePoolCount = pool.getActiveCount();
while (!pool.awaitTermination(5, TimeUnit.HOURS)) {
int newActivePoolCount = pool.getActiveCount();
if (activePoolCount == newActivePoolCount) {
logger.warn("Aborting the last {} jobs of the crawl, taking too long", newActivePoolCount);
pool.shutDownNow();
} else {
activePoolCount = newActivePoolCount;
}
}
}
catch (Exception ex) {
logger.warn("Exception in crawler", ex);
}
finally {
heartbeat.shutDown();