You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by dp...@apache.org on 2026/06/04 09:08:37 UTC
(stormcrawler) branch thread-safety created (now c05d7129)
This is an automated email from the ASF dual-hosted git repository.
dpol1 pushed a change to branch thread-safety
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
at c05d7129 fix: replace non-thread-safe SimpleDateFormat and shared Matcher instances
This branch includes the following new commits:
new c05d7129 fix: replace non-thread-safe SimpleDateFormat and shared Matcher instances
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(stormcrawler) 01/01: fix: replace non-thread-safe SimpleDateFormat and shared Matcher instances
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpol1 pushed a commit to branch thread-safety
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit c05d712921d95e2896c9334359ab92948360c4c9
Author: Davide Polato <dp...@apache.org>
AuthorDate: Thu Jun 4 11:00:15 2026 +0200
fix: replace non-thread-safe SimpleDateFormat and shared Matcher instances
- RefreshTag: replace static shared Matcher with Pattern + per-call Matcher
- FileResponse: replace SimpleDateFormat with DateTimeFormatter; use
try-with-resources for FileInputStream
- CookieConverter: replace SimpleDateFormat with DateUtils.parseDate();
fix duplicate setExpiryDate() call; add null guard on parsed date
---
.../stormcrawler/protocol/file/FileResponse.java | 13 +++++------
.../apache/stormcrawler/util/CookieConverter.java | 26 ++++++++++------------
.../org/apache/stormcrawler/util/RefreshTag.java | 9 ++++----
3 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
index 80ffc00d..47504a64 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
@@ -24,8 +24,6 @@ import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.Date;
import java.util.Locale;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHeaders;
@@ -37,8 +35,9 @@ import org.slf4j.LoggerFactory;
public class FileResponse {
- static final SimpleDateFormat dateFormat =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
+ static final java.time.format.DateTimeFormatter DATE_FORMATTER =
+ java.time.format.DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US)
+ .withZone(java.time.ZoneId.systemDefault());
static final org.slf4j.Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -101,8 +100,8 @@ public class FileResponse {
return;
}
- try {
- content = IOUtils.toByteArray(new FileInputStream(file), size);
+ try (FileInputStream fis = new FileInputStream(file)) {
+ content = IOUtils.toByteArray(fis, size);
} catch (IOException | IllegalArgumentException e) {
LOG.error("Exception while fetching file response {} ", file.getPath(), e);
statusCode = HttpStatus.SC_METHOD_FAILURE;
@@ -122,7 +121,7 @@ public class FileResponse {
}
private static String formatDate(long date) {
- return dateFormat.format(new Date(date));
+ return DATE_FORMATTER.format(java.time.Instant.ofEpochMilli(date));
}
private byte[] generateSitemap(File dir) {
diff --git a/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java b/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java
index c4b14c7b..5592fd50 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java
@@ -18,8 +18,6 @@
package org.apache.stormcrawler.util;
import java.net.URL;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -30,8 +28,8 @@ import org.apache.http.impl.cookie.BasicClientCookie;
/** Helper to extract cookies from cookies string. */
public class CookieConverter {
- private static final SimpleDateFormat DATE_FORMAT =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH);
+ private static final org.slf4j.Logger LOG =
+ org.slf4j.LoggerFactory.getLogger(CookieConverter.class);
/**
* Get a list of cookies based on the cookies string taken from response header and the target
@@ -110,17 +108,17 @@ public class CookieConverter {
// check expiration
if (expires != null) {
try {
- Date expirationDate = DATE_FORMAT.parse(expires);
- cookie.setExpiryDate(expirationDate);
-
- // check that it hasn't expired?
- if (cookie.isExpired(new Date())) {
- continue;
+ Date expirationDate = org.apache.http.client.utils.DateUtils.parseDate(expires);
+ if (expirationDate != null) {
+ cookie.setExpiryDate(expirationDate);
+
+ // check that it hasn't expired?
+ if (cookie.isExpired(new Date())) {
+ continue;
+ }
}
-
- cookie.setExpiryDate(expirationDate);
- } catch (ParseException e) {
- // ignore exceptions
+ } catch (Exception e) {
+ LOG.debug("Could not parse cookie expiry date: {}", expires, e);
}
}
diff --git a/core/src/main/java/org/apache/stormcrawler/util/RefreshTag.java b/core/src/main/java/org/apache/stormcrawler/util/RefreshTag.java
index b4532d7f..65953d72 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/RefreshTag.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/RefreshTag.java
@@ -28,8 +28,8 @@ import org.jsoup.select.QueryParser;
// Utility class used to extract refresh tags from HTML pages
public abstract class RefreshTag {
- private static final Matcher MATCHER =
- Pattern.compile("^.*;\\s*URL='?(.+?)'?$", Pattern.CASE_INSENSITIVE).matcher("");
+ private static final Pattern PATTERN =
+ Pattern.compile("^.*;\\s*URL='?(.+?)'?$", Pattern.CASE_INSENSITIVE);
private static final Evaluator EVALUATOR =
QueryParser.parse("meta[http-equiv~=(?i)refresh][content]");
@@ -42,8 +42,9 @@ public abstract class RefreshTag {
// 0;URL=http://www.apollocolors.com/site
try {
- if (MATCHER.reset(value).matches()) {
- return MATCHER.group(1);
+ Matcher matcher = PATTERN.matcher(value);
+ if (matcher.matches()) {
+ return matcher.group(1);
}
} catch (Exception e) {
}