Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 05:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Terminate crawler after a few hours of no progress

Browse Source
This commit is contained in:
Viktor Lofgren 2023-10-26 12:49:28 +02:00
parent 0f637fb722
commit a497e4c920
2 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
code/libraries/blocking-thread-pool/src/main/java/nu/marginalia/util/SimpleBlockingThreadPool.java
View File

@ -3,6 +3,7 @@ package nu.marginalia.util;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
@ -47,12 +48,15 @@ public class SimpleBlockingThreadPool {
this.shutDown = true; this.shutDown = true;
} }
public void shutDownNow() { public void shutDownNow() throws InterruptedException {
this.shutDown = true; this.shutDown = true;
tasks.clear(); tasks.clear();
for (Thread worker : workers) { for (Thread worker : workers) {
worker.interrupt(); worker.interrupt();
} }
for (Thread worker : workers) {
worker.join(Duration.ofMinutes(5));
}
} }
private void worker() { private void worker() {

16
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java
View File

@ -151,12 +151,20 @@ public class CrawlerMain {
logger.info("Shutting down the pool, waiting for tasks to complete..."); logger.info("Shutting down the pool, waiting for tasks to complete...");
pool.shutDown(); pool.shutDown();
do { int activePoolCount = pool.getActiveCount();
System.out.println("Waiting for pool to terminate... " + pool.getActiveCount() + " remaining");
} while (!pool.awaitTermination(60, TimeUnit.SECONDS)); while (!pool.awaitTermination(5, TimeUnit.HOURS)) {
int newActivePoolCount = pool.getActiveCount();
if (activePoolCount == newActivePoolCount) {
logger.warn("Aborting the last {} jobs of the crawl, taking too long", newActivePoolCount);
pool.shutDownNow();
} else {
activePoolCount = newActivePoolCount;
}
}
} }
catch (Exception ex) { catch (Exception ex) {
logger.warn("Exception in crawler", ex);
} }
finally { finally {
heartbeat.shutDown(); heartbeat.shutDown();