mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-02-24 05:18:58 +00:00
(minor) Fix accidental commit errors
This commit is contained in:
parent
e9854f194c
commit
162fc25ebc
@ -20,6 +20,7 @@ public class AnchorTagsSourceFactory {
|
|||||||
private final int nodeId;
|
private final int nodeId;
|
||||||
private final HikariDataSource dataSource;
|
private final HikariDataSource dataSource;
|
||||||
private static final Logger logger = LoggerFactory.getLogger(AnchorTagsSourceFactory.class);
|
private static final Logger logger = LoggerFactory.getLogger(AnchorTagsSourceFactory.class);
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
public AnchorTagsSourceFactory(HikariDataSource dataSource,
|
public AnchorTagsSourceFactory(HikariDataSource dataSource,
|
||||||
ProcessConfiguration config)
|
ProcessConfiguration config)
|
||||||
@ -30,8 +31,15 @@ public class AnchorTagsSourceFactory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public AnchorTagsSource create() throws SQLException {
|
public AnchorTagsSource create() throws SQLException {
|
||||||
|
try {
|
||||||
return create(getRelevantDomainsByNodeAffinity());
|
return create(getRelevantDomainsByNodeAffinity());
|
||||||
}
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
// likely a test environment
|
||||||
|
logger.warn("Failed to create anchor tags source", e);
|
||||||
|
return domain -> new DomainLinks();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public AnchorTagsSource create(List<EdgeDomain> relevantDomains) throws SQLException {
|
public AnchorTagsSource create(List<EdgeDomain> relevantDomains) throws SQLException {
|
||||||
if (!Files.exists(atagsPath)) {
|
if (!Files.exists(atagsPath)) {
|
||||||
|
@ -105,6 +105,8 @@ public class CrawlingThenConvertingIntegrationTest {
|
|||||||
|
|
||||||
CrawledDomain crawlData = crawl(specs);
|
CrawledDomain crawlData = crawl(specs);
|
||||||
|
|
||||||
|
System.out.println(crawlData);
|
||||||
|
|
||||||
assertEquals("REDIRECT", crawlData.crawlerStatus);
|
assertEquals("REDIRECT", crawlData.crawlerStatus);
|
||||||
assertEquals("www.marginalia.nu", crawlData.redirectDomain);
|
assertEquals("www.marginalia.nu", crawlData.redirectDomain);
|
||||||
assertTrue(crawlData.doc.isEmpty());
|
assertTrue(crawlData.doc.isEmpty());
|
||||||
|
@ -127,7 +127,7 @@ public class HttpFetcherImpl implements HttpFetcher {
|
|||||||
EdgeUrl requestUrl = new EdgeUrl(rsp.request().url().toString());
|
EdgeUrl requestUrl = new EdgeUrl(rsp.request().url().toString());
|
||||||
|
|
||||||
if (!Objects.equals(requestUrl.domain, url.domain)) {
|
if (!Objects.equals(requestUrl.domain, url.domain)) {
|
||||||
return new ProbeResultRedirect(url.domain);
|
return new ProbeResultRedirect(requestUrl.domain);
|
||||||
}
|
}
|
||||||
return new ProbeResultOk(requestUrl);
|
return new ProbeResultOk(requestUrl);
|
||||||
}
|
}
|
||||||
|
@ -42,6 +42,7 @@ class CrawledDocumentParquetRecordFileWriterTest {
|
|||||||
Instant.now(),
|
Instant.now(),
|
||||||
"text/html",
|
"text/html",
|
||||||
"hello world".getBytes(),
|
"hello world".getBytes(),
|
||||||
|
null,
|
||||||
null, null);
|
null, null);
|
||||||
|
|
||||||
// Write the record to a file
|
// Write the record to a file
|
||||||
|
@ -96,7 +96,7 @@ public class ScreenshotCaptureToolMain {
|
|||||||
private static byte[] fetchDomain(HttpClient client, EdgeDomain domain) {
|
private static byte[] fetchDomain(HttpClient client, EdgeDomain domain) {
|
||||||
try {
|
try {
|
||||||
Map<String, Object> requestData = Map.of(
|
Map<String, Object> requestData = Map.of(
|
||||||
"url", domain.toRootUrlHttp().toString(),
|
"url", domain.toRootUrl().toString(),
|
||||||
"options",
|
"options",
|
||||||
Map.of("fullPage", false,
|
Map.of("fullPage", false,
|
||||||
"type", "png"),
|
"type", "png"),
|
||||||
|
Loading…
Reference in New Issue
Block a user