Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 04:58:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(converter) Adding some logging around the simple processing track to investigate an issue with the converter stalling

Browse Source
This commit is contained in:
Viktor Lofgren 2025-01-26 12:02:00 +01:00
parent 2b9d2985ba
commit db99242db2
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code/processes/converting-process/java/nu/marginalia/converting/processor/DomainProcessor.java
View File

@ -194,6 +194,9 @@ public class DomainProcessor {
@Override @Override
public Iterator<ProcessedDocument> getDocumentsStream() { public Iterator<ProcessedDocument> getDocumentsStream() {
return iteratorFactory.create((taskConsumer) -> { return iteratorFactory.create((taskConsumer) -> {
logger.info("Simple Processing: {}", domain);
while (dataStream.hasNext()) while (dataStream.hasNext())
{ {
if (!(dataStream.next() instanceof CrawledDocument doc)) if (!(dataStream.next() instanceof CrawledDocument doc))
@ -218,6 +221,8 @@ public class DomainProcessor {
return processedDoc; return processedDoc;
}); });
} }
logger.info("Finished Simple Processing: {}", domain);
}); });
} }