From b5fa5fbecd695ca0e7c3083dd2316dd14a69a120 Mon Sep 17 00:00:00 2001 From: SCORP711 Date: Mon, 12 Oct 2026 20:55:12 +0200 Subject: [PATCH] #2211 Migrate core to HttpClient 5 and preserve parsing behavior # Conflicts: # pom.xml --- THIRD-PARTY.txt | 1 + core/pom.xml | 7 +- .../stormcrawler/bolt/FeedParserBolt.java | 2 +- .../stormcrawler/bolt/JSoupParserBolt.java | 2 +- .../stormcrawler/bolt/SimpleFetcherBolt.java | 2 +- .../stormcrawler/bolt/SiteMapParserBolt.java | 2 +- .../stormcrawler/fetcher/FetchOutcomes.java | 2 +- .../filtering/basic/BasicURLNormalizer.java | 9 +- .../persistence/AdaptiveScheduler.java | 2 +- .../protocol/HttpRobotRulesParser.java | 2 +- .../protocol/file/FileResponse.java | 4 +- .../protocol/okhttp/HttpProtocol.java | 6 +- .../util/CharsetIdentification.java | 2 +- .../stormcrawler/util/CookieConverter.java | 75 ++++++++-- .../stormcrawler/bolt/FeedParserBoltTest.java | 2 +- .../bolt/SiteMapParserBoltTest.java | 2 +- .../filtering/BasicURLNormalizerTest.java | 31 ++++ .../persistence/AdaptiveSchedulerTest.java | 2 +- .../util/CookieConverterTest.java | 132 ++++++++++++++---- docs/src/main/asciidoc/index.adoc | 2 + docs/src/main/asciidoc/migrating.adoc | 18 +++ .../apache/stormcrawler/tika/ParserBolt.java | 2 +- .../stormcrawler/tika/ParserBoltTest.java | 2 +- .../ParserBoltWhitelistDetectionTest.java | 2 +- .../stormcrawler/warc/WARCRecordFormat.java | 2 +- .../warc/WARCRecordFormatTest.java | 2 +- pom.xml | 1 + 27 files changed, 249 insertions(+), 69 deletions(-) create mode 100644 docs/src/main/asciidoc/migrating.adoc diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt index fe7de2b92..3acfd88a7 100644 --- a/THIRD-PARTY.txt +++ b/THIRD-PARTY.txt @@ -31,6 +31,7 @@ List of third-party dependencies grouped by their license type. * Apache Commons IO (commons-io:commons-io:2.22.0 - https://commons.apache.org/proper/commons-io/) * Apache Commons Lang (org.apache.commons:commons-lang3:3.21.0 - https://commons.apache.org/proper/commons-lang/) * Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/) + * Apache Commons Logging (commons-logging:commons-logging:1.3.0 - https://commons.apache.org/proper/commons-logging/) * Apache Commons Logging (commons-logging:commons-logging:1.3.6 - https://commons.apache.org/proper/commons-logging/) * Apache Commons Logging (commons-logging:commons-logging:1.4.0 - https://commons.apache.org/proper/commons-logging/) * Apache Commons Math (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/) diff --git a/core/pom.xml b/core/pom.xml index 7b966ff5f..eab9ba85f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -41,7 +41,6 @@ under the License. 1.23.2 78.3 2.12.2 - 4.5.14 2.7 3.21.0 3.13.2 @@ -215,9 +214,9 @@ under the License. - org.apache.httpcomponents - httpclient - ${httpclient.version} + org.apache.httpcomponents.core5 + httpcore5 + ${httpcore5.version} diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java index 1591c14d9..7b1a1dfff 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java @@ -36,7 +36,7 @@ import java.util.Map; import java.util.TimeZone; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index dc8c32c92..a7ef8345b 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java index bdd03eb2f..ecc321317 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java @@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.Config; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java index bd8b15cb4..15830d048 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java @@ -44,7 +44,7 @@ import java.util.TimeZone; import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; diff --git a/core/src/main/java/org/apache/stormcrawler/fetcher/FetchOutcomes.java b/core/src/main/java/org/apache/stormcrawler/fetcher/FetchOutcomes.java index 46b52d3fe..d7d6fe701 100644 --- a/core/src/main/java/org/apache/stormcrawler/fetcher/FetchOutcomes.java +++ b/core/src/main/java/org/apache/stormcrawler/fetcher/FetchOutcomes.java @@ -20,7 +20,7 @@ import java.io.InterruptedIOException; import java.net.UnknownHostException; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.bolt.FetcherBolt; diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 4e14119e5..af4eb5a2a 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -37,8 +37,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.net.URLEncodedUtils; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.filtering.URLFilter; import org.apache.stormcrawler.util.URLUtil; @@ -289,7 +289,10 @@ private String processQueryElements(String urlToFilter) { return urlToFilter; } - List pairs = URLEncodedUtils.parse(query, StandardCharsets.UTF_8); + // Keep parse and format: WWWFormCodec changes semicolon splitting and tilde/star + // encoding. + List pairs = + URLEncodedUtils.parse(query, StandardCharsets.UTF_8, '&', ';'); Iterator pairsIterator = pairs.iterator(); while (pairsIterator.hasNext()) { NameValuePair param = pairsIterator.next(); diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java b/core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java index 28d50bcfc..e21b5d24c 100644 --- a/core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java +++ b/core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.Optional; import java.util.TimeZone; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.parse.filter.MD5SignatureParseFilter; diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index 531e744aa..f4ba6c9bd 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -28,7 +28,7 @@ import java.util.Locale; import java.util.Set; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.Config; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.util.ConfUtils; diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java index 028984deb..b65dfa257 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java @@ -26,8 +26,8 @@ import java.nio.charset.StandardCharsets; import java.util.Locale; import org.apache.commons.io.IOUtils; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpStatus; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.util.URLUtil; diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index f742800b4..952b42bdc 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -71,8 +71,7 @@ import okio.BufferedSource; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableObject; -import org.apache.http.HttpHeaders; -import org.apache.http.cookie.Cookie; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.Config; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; @@ -86,6 +85,7 @@ import org.apache.stormcrawler.proxy.SCProxy; import org.apache.stormcrawler.util.ConfUtils; import org.apache.stormcrawler.util.CookieConverter; +import org.apache.stormcrawler.util.CookieConverter.Cookie; import org.apache.stormcrawler.util.URLUtil; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; @@ -506,7 +506,7 @@ private void addCookiesToRequest(Builder rb, String url, Metadata md, boolean se CookieConverter.getCookies( cookieStrings, getCookieOrigin(md, url), URLUtil.toURL(url)); for (Cookie c : cookies) { - rb.addHeader("Cookie", c.getName() + "=" + c.getValue()); + rb.addHeader("Cookie", c.name() + "=" + c.value()); } } catch (MalformedURLException e) { // Bad url , nothing to do } diff --git a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java index 9fc778dd3..a3240eaae 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java @@ -30,7 +30,7 @@ import java.util.regex.Pattern; import org.apache.commons.io.ByteOrderMark; import org.apache.commons.io.input.BOMInputStream; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.stormcrawler.Metadata; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; diff --git a/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java b/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java index e8501b816..4b7d24573 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java @@ -20,22 +20,52 @@ import crawlercommons.domains.EffectiveTldFinder; import java.net.IDN; import java.net.URL; +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Locale; -import org.apache.http.cookie.Cookie; -import org.apache.http.impl.cookie.BasicClientCookie; +import java.util.TimeZone; /** Helper to extract cookies from cookies string. */ public class CookieConverter { + /** An immutable cookie accepted for a target URL. */ + public record Cookie( + String name, + String value, + String domain, + String path, + boolean secure, + Instant expiry) {} + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(CookieConverter.class); /** Maximum length of a domain name in ASCII characters, see RFC 1034. */ private static final int MAX_DOMAIN_LENGTH = 253; + private static final String[] COOKIE_DATE_PATTERNS = { + "EEE, dd MMM yyyy HH:mm:ss zzz", "EEE, dd-MMM-yy HH:mm:ss zzz", "EEE MMM d HH:mm:ss yyyy" + }; + + private static final Date COOKIE_DATE_PIVOT = Date.from(Instant.parse("2000-01-01T00:00:00Z")); + + private static final ThreadLocal COOKIE_DATE_FORMATS = + ThreadLocal.withInitial( + () -> { + SimpleDateFormat[] formats = + new SimpleDateFormat[COOKIE_DATE_PATTERNS.length]; + for (int i = 0; i < COOKIE_DATE_PATTERNS.length; i++) { + formats[i] = new SimpleDateFormat(COOKIE_DATE_PATTERNS[i], Locale.US); + formats[i].setTimeZone(TimeZone.getTimeZone("GMT")); + formats[i].set2DigitYearStart(COOKIE_DATE_PIVOT); + } + return formats; + }); + /** * Get a list of cookies based on the cookies string taken from response header and the target * url. As the host which set the cookies is unknown, cookies without a domain attribute are @@ -115,8 +145,6 @@ public static List getCookies(String[] cookiesStrings, URL originURL, UR } } - BasicClientCookie cookie = new BasicClientCookie(name, value); - // check domain if (domain != null && !domain.isBlank()) { final String scope = normaliseDomain(domain); @@ -125,8 +153,6 @@ public static List getCookies(String[] cookiesStrings, URL originURL, UR continue; } - cookie.setDomain(domain); - // a domain which can not own subdomains, i.e. a public suffix such as // "com" or "co.uk" or a single label, binds the cookie to the host itself final boolean subdomains = allowsSubdomains(scope); @@ -167,8 +193,6 @@ public static List getCookies(String[] cookiesStrings, URL originURL, UR // check path if (path != null) { - cookie.setPath(path); - if (!path.equals("") && !path.equals("/") && !targetURL.getPath().startsWith(path)) { @@ -183,8 +207,6 @@ public static List getCookies(String[] cookiesStrings, URL originURL, UR // check secure if (secure) { - cookie.setSecure(secure); - if (!targetURL.getProtocol().equalsIgnoreCase("https")) { LOG.debug("Skipping cookie {}: secure and {} is not https", name, targetURL); continue; @@ -192,14 +214,13 @@ public static List getCookies(String[] cookiesStrings, URL originURL, UR } // check expiration + Instant expirationDate = null; if (expires != null) { try { - Date expirationDate = org.apache.http.client.utils.DateUtils.parseDate(expires); + expirationDate = parseCookieExpiry(expires); if (expirationDate != null) { - cookie.setExpiryDate(expirationDate); - // check that it hasn't expired? - if (cookie.isExpired(new Date())) { + if (!expirationDate.isAfter(Instant.now())) { LOG.debug("Skipping cookie {}: expired on {}", name, expires); continue; } @@ -210,12 +231,36 @@ public static List getCookies(String[] cookiesStrings, URL originURL, UR } // attach additional infos to cookie - list.add(cookie); + list.add( + new Cookie( + name, + value, + domain == null || domain.isBlank() + ? null + : domain.toLowerCase(Locale.ROOT), + path, + secure, + expirationDate)); } return list; } + static Instant parseCookieExpiry(String expires) { + // Preserve legacy date parsing so expired cookies do not become session cookies. + String value = expires; + if (value.length() > 1 && value.startsWith("'") && value.endsWith("'")) { + value = value.substring(1, value.length() - 1); + } + for (SimpleDateFormat format : COOKIE_DATE_FORMATS.get()) { + Date legacyDate = format.parse(value, new ParsePosition(0)); + if (legacyDate != null) { + return legacyDate.toInstant(); + } + } + return null; + } + /** * Normalises the value of a cookie domain attribute into the domain it scopes the cookie to. * diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java index a7c2a954d..c39f60a4a 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java @@ -21,7 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java index 4ed1edda7..4d8616d53 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java @@ -26,7 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Values; import org.apache.stormcrawler.Constants; diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 5df8b97a7..1791bbcc3 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -44,6 +44,37 @@ class BasicURLNormalizerTest { List queryParamsToFilter = Arrays.asList("a", "foo"); + @Test + void testMixedQuerySeparatorsAndEncoding() { + URLFilter filter = createFilter(List.of("tracking")); + assertEquals( + "http://example.com/?a=2&b=1&c=x+y+z&d", + filter.filter(null, new Metadata(), "http://example.com/?b=1;a=2&c=x+y%20z&d")); + assertEquals( + "http://example.com/?a=&b=x%3By%26z&b=%2B&flag", + filter.filter( + null, new Metadata(), "http://example.com/?flag;b=x%3By%26z&b=%2B;a=")); + assertEquals( + "http://example.com/?a=%C3%A9&b=%E2%82%AC", + filter.filter(null, new Metadata(), "http://example.com/?b=%E2%82%AC;a=%C3%A9")); + } + + @Test + void testQueryFormattingKeepsLegacyEncoding() { + URLFilter filter = createFilter(List.of("tracking")); + assertEquals( + "http://example.com/?a=~%21%27%28%29*&b=2", + filter.filter(null, new Metadata(), "http://example.com/?b=2&a=~!'()*")); + } + + @Test + void testRemoveSemicolonSeparatedQueryParameters() { + URLFilter filter = createFilter(queryParamsToFilter); + assertEquals( + "http://example.com/?keep=yes", + filter.filter(null, new Metadata(), "http://example.com/?foo=bar;keep=yes&a=1")); + } + private URLFilter createFilter(boolean removeAnchor, boolean checkValidURI) { ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance); filterParams.put("removeAnchorPart", Boolean.valueOf(removeAnchor)); diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java index d754d326a..43b3a3628 100644 --- a/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java @@ -31,7 +31,7 @@ import java.util.Optional; import java.util.TimeZone; import org.apache.commons.lang3.time.DateUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.stormcrawler.Metadata; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java index e5b651f33..8d03c6cd2 100644 --- a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java +++ b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java @@ -19,15 +19,97 @@ import java.net.MalformedURLException; import java.net.URL; +import java.time.Instant; import java.util.List; -import org.apache.http.cookie.Cookie; +import org.apache.stormcrawler.util.CookieConverter.Cookie; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class CookieConverterTest { + @ParameterizedTest + @ValueSource( + strings = { + "Wed, 09 Jun 2021 10:18:14 GMT", + "Wednesday, 09-Jun-21 10:18:14 GMT", + "Wed Jun 9 10:18:14 2021", + "'Wed, 09 Jun 2021 10:18:14 GMT'" + }) + void testExpiredCookieDateFormats(String expires) { + List cookies = + CookieConverter.getCookies( + new String[] {"sid=value; Expires=" + expires}, + getUrl(unsecuredUrl), + getUrl(unsecuredUrl)); + Assertions.assertTrue(cookies.isEmpty()); + } + + @Test + void testInvalidExpiryRemainsASessionCookie() { + List cookies = + CookieConverter.getCookies( + new String[] {"sid=value; Expires=invalid"}, + getUrl(unsecuredUrl), + getUrl(unsecuredUrl)); + Assertions.assertEquals(1, cookies.size()); + Assertions.assertNull(cookies.get(0).expiry()); + } + + @ParameterizedTest + @ValueSource( + strings = { + "Tue, 11 Apr 2117 07:13:39 -0000", + "Sun, 11 Apr 2117 07:13:39 GMT", + "Sun Apr 11 07:13:39 2117", + "'Sun Apr 11 07:13:39 2117'", + "Sun, 11 Apr 2117 09:13:39 +0200" + }) + void testFutureCookieKeepsItsExpiry(String expires) { + List cookies = + CookieConverter.getCookies( + new String[] {"sid=value; Expires=" + expires}, + getUrl(unsecuredUrl), + getUrl(unsecuredUrl)); + Assertions.assertEquals(1, cookies.size()); + Assertions.assertEquals(Instant.parse("2117-04-11T07:13:39Z"), cookies.get(0).expiry()); + } + + @Test + void testCookieExpiryPreservesLegacyTimezoneAndTwoDigitYearParsing() { + Assertions.assertEquals( + Instant.parse("2021-06-09T18:18:14Z"), + CookieConverter.parseCookieExpiry("Wed, 09 Jun 2021 10:18:14 PST")); + Assertions.assertEquals( + Instant.parse("0021-06-07T10:18:14Z"), + CookieConverter.parseCookieExpiry("Wed, 09 Jun 21 10:18:14 GMT")); + } + private static String securedUrl = "https://someurl.com"; + @Test + void testCookieRecordRetainsAttributes() { + List cookies = + CookieConverter.getCookies( + new String[] { + "sid=value; Domain=SomeUrl.COM; Path=/; Secure; Expires=Sun Apr 11" + + " 07:13:39 2117" + }, + getUrl(securedUrl), + getUrl(securedUrl)); + Assertions.assertEquals( + List.of( + new Cookie( + "sid", + "value", + "someurl.com", + "/", + true, + Instant.parse("2117-04-11T07:13:39Z"))), + cookies); + } + private static String unsecuredUrl = "http://someurl.com"; private static String dummyCookieHeader = "nice tasty test cookie header!"; @@ -45,9 +127,9 @@ void testSimpleCookieAndUrl() { cookiesStrings, getUrl(unsecuredUrl), getUrl(unsecuredUrl)); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -67,9 +149,9 @@ void testNotExpiredCookie() { cookiesStrings, getUrl(unsecuredUrl), getUrl(unsecuredUrl)); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -102,9 +184,9 @@ void testValidPath() { cookiesStrings, getUrl(unsecuredUrl), getUrl(unsecuredUrl + "/somepage")); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -118,9 +200,9 @@ void testValidPath2() { cookiesStrings, getUrl(unsecuredUrl), getUrl(unsecuredUrl)); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -135,9 +217,9 @@ void testValidPath3() { cookiesStrings, getUrl(unsecuredUrl), getUrl(unsecuredUrl + "/someFolder")); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -154,9 +236,9 @@ void testValidPath4() { getUrl(unsecuredUrl + "/someFolder/SomeOtherFolder")); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -188,9 +270,9 @@ void testValidDomain() { getUrl(unsecuredUrl + "/someFolder/SomeOtherFolder")); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -238,9 +320,9 @@ void testSecurFlagHttpS() { getUrl(securedUrl + "/someFolder/SomeOtherFolder")); Assertions.assertEquals(1, result.size(), "Target url is secured - Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -262,9 +344,9 @@ void testFullCookie() { getUrl(securedUrl + "/someFolder/SomeOtherFolder")); Assertions.assertEquals(1, result.size(), "Should have 1 cookie"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); } @Test @@ -295,17 +377,15 @@ void test2Cookies() { getUrl(securedUrl + "/someFolder/SomeOtherFolder")); Assertions.assertEquals(2, result.size(), "Should have 2 cookies"); Assertions.assertEquals( - dummyCookieHeader, result.get(0).getName(), "Cookie header should be as defined"); + dummyCookieHeader, result.get(0).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue, result.get(0).getValue(), "Cookie value should be as defined"); + dummyCookieValue, result.get(0).value(), "Cookie value should be as defined"); Assertions.assertEquals( dummyCookieHeader + "2", - result.get(1).getName(), + result.get(1).name(), "Cookie header should be as defined"); Assertions.assertEquals( - dummyCookieValue + "2", - result.get(1).getValue(), - "Cookie value should be as defined"); + dummyCookieValue + "2", result.get(1).value(), "Cookie value should be as defined"); } @Test @@ -678,7 +758,7 @@ void cookieHeaderWithoutANameIsSkipped() { CookieConverter.getCookies( cookiesStrings, getUrl(unsecuredUrl), getUrl(unsecuredUrl)); Assertions.assertEquals(1, result.size(), "Only the usable cookie is kept"); - Assertions.assertEquals("sid", result.get(0).getName()); + Assertions.assertEquals("sid", result.get(0).name()); } @Test diff --git a/docs/src/main/asciidoc/index.adoc b/docs/src/main/asciidoc/index.adoc index 8a71db356..cb639f6f7 100644 --- a/docs/src/main/asciidoc/index.adoc +++ b/docs/src/main/asciidoc/index.adoc @@ -20,6 +20,8 @@ include::overview.adoc[] include::quick-start.adoc[] +include::migrating.adoc[] + include::architecture.adoc[] include::internals.adoc[] diff --git a/docs/src/main/asciidoc/migrating.adoc b/docs/src/main/asciidoc/migrating.adoc new file mode 100644 index 000000000..7e07ec3b1 --- /dev/null +++ b/docs/src/main/asciidoc/migrating.adoc @@ -0,0 +1,18 @@ +//// +Licensed under the Apache License, Version 2.0 (the "License"); +You may not use this file except in compliance with the License. +You may obtain a copy of the License at: +https://www.apache.org/licenses/LICENSE-2.0 +//// +== Migrating to 4.0.0 + +Both overloads of `CookieConverter.getCookies(...)` now return immutable +`org.apache.stormcrawler.util.CookieConverter.Cookie` records instead of +HttpClient cookies. Update the cookie type import and use the record accessors +`name()`, `value()`, `domain()`, `path()`, `secure()` and `expiry()`. +The expiry is a `java.time.Instant`, or `null` for a session cookie. + +Core now depends only on HttpCore 5, not HttpClient. Applications using +HttpClient classes through core's former transitive dependency must declare +their own HttpClient dependency. The legacy OpenSearch module still uses +HttpClient 4 through its REST client. diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index 3d355171c..78876a695 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -44,8 +44,8 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParserFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.html.dom.HTMLDocumentImpl; -import org.apache.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java index b8234fc4c..b411b0ef3 100644 --- a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java @@ -24,7 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Constants; diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltWhitelistDetectionTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltWhitelistDetectionTest.java index 85e6d133b..7f4e34b1b 100644 --- a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltWhitelistDetectionTest.java +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltWhitelistDetectionTest.java @@ -22,7 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index a7111f98d..7bece3d88 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -40,7 +40,7 @@ import org.apache.commons.codec.binary.Base32; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.hdfs.bolt.format.RecordFormat; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index 11f9682b1..121578d4b 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -29,7 +29,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; -import org.apache.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.ProtocolResponse; diff --git a/pom.xml b/pom.xml index 2e24ae53a..15e30c91b 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ under the License. 2.22.0 1.28.0 1.23.0 + 5.4.3 33.7.2-jre 6.2 1.21.4