Mirror/MarginaliaSearch
1
0
Fork 0
mirror of https://github.com/MarginaliaSearch/MarginaliaSearch.git synced 2025-02-23 21:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(crawler) Strip W/-prefix from the etag when supplied as If-None-Match

Browse Source
This commit is contained in:
Viktor Lofgren 2024-04-22 14:31:05 +02:00
parent 6b88db10ad
commit c9f029c214
1 changed files with 33 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
code/processes/crawling-process/java/nu/marginalia/crawl/retreival/fetcher/ContentTags.java
View File

@ -18,7 +18,38 @@ public record ContentTags(String etag, String lastMod) {
/** Paints the tags onto the request builder. */
public void paint(Request.Builder getBuilder) {
if (etag != null) getBuilder.addHeader("If-None-Match", etag);
if (lastMod != null) getBuilder.addHeader("If-Modified-Since", lastMod);
System.out.println(ifNoneMatch() + " " + ifModifiedSince());
if (etag != null) {
getBuilder.addHeader("If-None-Match", ifNoneMatch());
}
if (lastMod != null) {
getBuilder.addHeader("If-Modified-Since", ifModifiedSince());
}
}
private String ifNoneMatch() {
// Remove the W/ prefix if it exists
//'W/' (case-sensitive) indicates that a weak validator is used. Weak etags are
// easy to generate, but are far less useful for comparisons. Strong validators
// are ideal for comparisons but can be very difficult to generate efficiently.
// Weak ETag values of two representations of the same resources might be semantically
// equivalent, but not byte-for-byte identical. This means weak etags prevent caching
// when byte range requests are used, but strong etags mean range requests can
// still be cached.
// - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
if (null != etag && etag.startsWith("W/")) {
return etag.substring(2);
} else {
return etag;
}
}
private String ifModifiedSince() {
return lastMod;
}
}