Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Fix rare ConcurrentModificationError due to HashSet

Browse Source
This commit is contained in:
Viktor Lofgren 2023-08-01 17:28:29 +02:00
parent 58556af6c7
commit e5c9791b14
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java
View File

@ -54,7 +54,7 @@ public class CrawlerMain {
private final Gson gson; private final Gson gson;
private final DumbThreadPool pool; private final DumbThreadPool pool;
private final Set<String> processingIds = new HashSet<>(); private final Map<String, String> processingIds = new ConcurrentHashMap<>();
private final CrawledDomainReader reader = new CrawledDomainReader(); private final CrawledDomainReader reader = new CrawledDomainReader();
final AbortMonitor abortMonitor = AbortMonitor.getInstance(); final AbortMonitor abortMonitor = AbortMonitor.getInstance();
@ -148,7 +148,7 @@ public class CrawlerMain {
// This shouldn't realistically happen, but if it does, we need to ignore it, otherwise // This shouldn't realistically happen, but if it does, we need to ignore it, otherwise
// we'd end crawling the same site twice and might end up writing to the same output // we'd end crawling the same site twice and might end up writing to the same output
// file from multiple threads with complete bit salad as a result. // file from multiple threads with complete bit salad as a result.
if (!processingIds.add(crawlingSpecification.id)) { if (processingIds.put(crawlingSpecification.id, "") != null) {
logger.error("Ignoring duplicate id: {}", crawlingSpecification.id); logger.error("Ignoring duplicate id: {}", crawlingSpecification.id);
continue; continue;
} }