Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-24 13:19:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Send "Accept"-headers when fetching documents, also indicate we prefer English results

Browse Source
This commit is contained in:
Viktor Lofgren 2024-06-02 16:30:34 +02:00
parent 206a7ce6c1
commit b4eac2516e
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/HttpFetcherImpl.java
View File

@ -183,6 +183,8 @@ public class HttpFetcherImpl implements HttpFetcher {
getBuilder.url(url.toString()) getBuilder.url(url.toString())
.addHeader("Accept-Encoding", "gzip") .addHeader("Accept-Encoding", "gzip")
.addHeader("Accept-Language", "en,*;q=0.5")
.addHeader("Accept", "text/html, application/xhtml+xml, */*;q=0.8")
.addHeader("User-agent", userAgentString); .addHeader("User-agent", userAgentString);
contentTags.paint(getBuilder); contentTags.paint(getBuilder);
@ -225,6 +227,7 @@ public class HttpFetcherImpl implements HttpFetcher {
getBuilder.url(url.toString()) getBuilder.url(url.toString())
.addHeader("Accept-Encoding", "gzip") .addHeader("Accept-Encoding", "gzip")
.addHeader("Accept", "text/*, */*;q=0.9")
.addHeader("User-agent", userAgentString); .addHeader("User-agent", userAgentString);
HttpFetchResult result = recorder.fetch(client, getBuilder.build()); HttpFetchResult result = recorder.fetch(client, getBuilder.build());