mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-02-24 21:29:00 +00:00
1d34224416
Look, this will make the git history look funny, but trimming unnecessary depth from the source tree is a very necessary sanity-preserving measure when dealing with a super-modularized codebase like this one. While it makes the project configuration a bit less conventional, it will save you several clicks every time you jump between modules. Which you'll do a lot, because it's *modul*ar. The src/main/java convention makes a lot of sense for a non-modular project though. This ain't that.
56 lines
2.6 KiB
Java
56 lines
2.6 KiB
Java
package nu.marginalia.crawling;
|
|
|
|
import lombok.SneakyThrows;
|
|
import nu.marginalia.crawl.retreival.RateLimitException;
|
|
import nu.marginalia.crawl.retreival.fetcher.ContentTags;
|
|
import nu.marginalia.crawl.retreival.fetcher.HttpFetcherImpl;
|
|
import nu.marginalia.crawling.body.DocumentBodyExtractor;
|
|
import nu.marginalia.crawling.body.DocumentBodyResult;
|
|
import nu.marginalia.crawl.retreival.fetcher.warc.WarcRecorder;
|
|
import nu.marginalia.crawling.body.ContentTypeLogic;
|
|
import nu.marginalia.model.EdgeUrl;
|
|
import org.junit.jupiter.api.Assertions;
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
import java.io.IOException;
|
|
import java.net.URISyntaxException;
|
|
|
|
class HttpFetcherTest {
|
|
|
|
@SneakyThrows
|
|
@Test
|
|
void testUrlPattern() {
|
|
ContentTypeLogic contentTypeLogic = new ContentTypeLogic();
|
|
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.txt")));
|
|
Assertions.assertTrue(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.bin")));
|
|
Assertions.assertTrue(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.tar.gz")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.htm")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.html")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log")));
|
|
Assertions.assertFalse(contentTypeLogic.isUrlLikeBinary(new EdgeUrl("https://marginalia.nu/log.php?id=1")));
|
|
}
|
|
|
|
@Test
|
|
void fetchUTF8() throws URISyntaxException, RateLimitException, IOException {
|
|
var fetcher = new HttpFetcherImpl("nu.marginalia.edge-crawler");
|
|
try (var recorder = new WarcRecorder()) {
|
|
var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu"), recorder, ContentTags.empty());
|
|
if (DocumentBodyExtractor.asString(result) instanceof DocumentBodyResult.Ok bodyOk) {
|
|
System.out.println(bodyOk.contentType());
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test
|
|
void fetchText() throws URISyntaxException, RateLimitException, IOException {
|
|
var fetcher = new HttpFetcherImpl("nu.marginalia.edge-crawler");
|
|
|
|
try (var recorder = new WarcRecorder()) {
|
|
var result = fetcher.fetchContent(new EdgeUrl("https://www.marginalia.nu/robots.txt"), recorder, ContentTags.empty());
|
|
if (DocumentBodyExtractor.asString(result) instanceof DocumentBodyResult.Ok bodyOk) {
|
|
System.out.println(bodyOk.contentType());
|
|
}
|
|
}
|
|
}
|
|
} |