mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-02-24 21:29:00 +00:00
e9854f194c
* Restructure the code to make a bit more sense * Store full headers in crawl data * Fix bug in retry-after header that assumed the timeout was in milliseconds, and then clamped it to a lower bound of 500ms, meaning this was almost always handled wrong
56 lines
2.6 KiB
Java
56 lines
2.6 KiB
Java
package nu.marginalia.crawling;
|
|
|
|
import lombok.SneakyThrows;
|
|
import nu.marginalia.crawl.fetcher.ContentTags;
|
|
import nu.marginalia.crawl.fetcher.HttpFetcher;
|
|
import nu.marginalia.crawl.fetcher.HttpFetcherImpl;
|
|
import nu.marginalia.crawl.fetcher.warc.WarcRecorder;
|
|
import nu.marginalia.model.EdgeUrl;
|
|
import nu.marginalia.model.body.ContentTypeLogic;
|
|
import nu.marginalia.model.body.DocumentBodyExtractor;
|
|
import nu.marginalia.model.body.DocumentBodyResult;
|
|
import org.junit.jupiter.api.Assertions;
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
import java.io.IOException;
|
|
import java.net.URISyntaxException;
|
|
|
|
class HttpFetcherTest {
|
|
|
|
@SneakyThrows
|
|
@Test
|
|
void testUrlPattern() {
|
|
ContentTypeLogic contentTypeLogic = new ContentTypeLogic();
|
|
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.txt")));
|
|
Assertions.assertTrue(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.bin")));
|
|
Assertions.assertTrue(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.tar.gz")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.htm")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.html")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.php?id=1")));
|
|
}
|
|
|
|
@Test
|
|
void fetchUTF8() throws URISyntaxException, HttpFetcherImpl.RateLimitException, IOException {
|
|
var fetcher = new HttpFetcherImpl("nu.marginalia.edge-crawler");
|
|
try (var recorder = new WarcRecorder()) {
|
|
var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu"), recorder, ContentTags.empty(), HttpFetcher.ProbeType.FULL);
|
|
if (DocumentBodyExtractor.asString(result) instanceof DocumentBodyResult.Ok bodyOk) {
|
|
System.out.println(bodyOk.contentType());
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test
|
|
void fetchText() throws URISyntaxException, HttpFetcherImpl.RateLimitException, IOException {
|
|
var fetcher = new HttpFetcherImpl("nu.marginalia.edge-crawler");
|
|
|
|
try (var recorder = new WarcRecorder()) {
|
|
var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu/robots.txt"), recorder, ContentTags.empty(), HttpFetcher.ProbeType.FULL);
|
|
if (DocumentBodyExtractor.asString(result) instanceof DocumentBodyResult.Ok bodyOk) {
|
|
System.out.println(bodyOk.contentType());
|
|
}
|
|
}
|
|
}
|
|
} |