Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 04:58:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(converter) Improve simple processing performance

Browse Source 
There was a regression introduced in the recent slop migration changes in  the performance of the simple conversion track.  This reverts the issue.
This commit is contained in:
Viktor Lofgren 2025-01-21 14:13:33 +01:00
parent 270cab874b
commit 088310e998
1 changed files with 21 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
code/processes/converting-process/java/nu/marginalia/converting/processor/DomainProcessor.java
View File

@ -193,13 +193,16 @@ public class DomainProcessor {
@Override
public Iterator<ProcessedDocument> getDocumentsStream() {
return dataStream.map((next) -> {
if (!(next instanceof CrawledDocument doc))
return Optional.empty();
return iteratorFactory.create((taskConsumer) -> {
while (dataStream.hasNext())
{
if (!(dataStream.next() instanceof CrawledDocument doc))
continue;
if (doc.url == null || !processedUrls.add(doc.url))
return Optional.empty();
continue;
taskConsumer.accept(() -> {
var processedDoc = documentProcessor.process(doc, domain.domain, externalDomainLinks, documentDecorator);
synchronized (deduplicator) {
@ -212,7 +215,9 @@ public class DomainProcessor {
10_000, externalDomainLinks.countForUrl(processedDoc.url));
}
return Optional.of(processedDoc);
return processedDoc;
});
}
});
}