You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by dp...@apache.org on 2026/06/18 18:12:28 UTC
(stormcrawler) branch main updated: #1951 refactor: replace linear proxy scan with lookup map in MultiProxyManager (#1954)
This is an automated email from the ASF dual-hosted git repository.
dpol1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new 6ecd0e0f #1951 refactor: replace linear proxy scan with lookup map in MultiProxyManager (#1954)
6ecd0e0f is described below
commit 6ecd0e0f1a9d982e7a935f40e4b1cf5191cdc3c2
Author: DONGGYU LEE <us...@gmail.com>
AuthorDate: Fri Jun 19 03:12:24 2026 +0900
#1951 refactor: replace linear proxy scan with lookup map in MultiProxyManager (#1954)
---
.../stormcrawler/proxy/MultiProxyManager.java | 20 ++++--
.../org/apache/stormcrawler/proxy/ProxyUtils.java | 15 -----
.../org/apache/stormcrawler/proxy/SCProxy.java | 55 ++++++++++++++---
.../org/apache/stormcrawler/proxy/SCProxyTest.java | 71 ++++++++++++++++++++++
4 files changed, 130 insertions(+), 31 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
index 7140388a..68a600d4 100644
--- a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
@@ -21,6 +21,8 @@ import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.Scanner;
@@ -44,6 +46,7 @@ public class MultiProxyManager implements ProxyManager {
private SCProxy[] proxies;
private ProxyRotation rotation;
private final AtomicInteger lastAccessedIndex = new AtomicInteger(0);
+ private Map<SCProxy, SCProxy> proxyLookupMap;
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
@@ -146,6 +149,11 @@ public class MultiProxyManager implements ProxyManager {
// assign proxies to class variable
this.proxies = fileProxies;
+
+ this.proxyLookupMap = new HashMap<>();
+ for (SCProxy proxy : this.proxies) {
+ this.proxyLookupMap.put(proxy, proxy);
+ }
}
public void configure(ProxyRotation rotation, String[] proxyList) throws RuntimeException {
@@ -170,6 +178,11 @@ public class MultiProxyManager implements ProxyManager {
// assign proxies to class variable
this.proxies = fileProxies;
+
+ this.proxyLookupMap = new HashMap<>();
+ for (SCProxy proxy : this.proxies) {
+ this.proxyLookupMap.put(proxy, proxy);
+ }
}
private SCProxy getRandom() {
@@ -215,12 +228,7 @@ public class MultiProxyManager implements ProxyManager {
}
private Optional<SCProxy> getConfiguredProxy(SCProxy proxy) {
- for (SCProxy configuredProxy : this.proxies) {
- if (ProxyUtils.isSameProxy(configuredProxy, proxy)) {
- return Optional.of(configuredProxy);
- }
- }
- return Optional.empty();
+ return Optional.ofNullable(proxyLookupMap.get(proxy));
}
@Override
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/ProxyUtils.java b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyUtils.java
index 9d3647d4..a87f73e6 100644
--- a/core/src/main/java/org/apache/stormcrawler/proxy/ProxyUtils.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyUtils.java
@@ -17,9 +17,6 @@
package org.apache.stormcrawler.proxy;
-import java.util.Locale;
-import java.util.Objects;
-
final class ProxyUtils {
static final int MIN_PORT = 1;
@@ -40,16 +37,4 @@ final class ProxyUtils {
+ "`");
}
}
-
- static boolean isSameProxy(SCProxy proxy, SCProxy otherProxy) {
- return Objects.equals(normalize(proxy.getProtocol()), normalize(otherProxy.getProtocol()))
- && Objects.equals(normalize(proxy.getAddress()), normalize(otherProxy.getAddress()))
- && Objects.equals(proxy.getPort(), otherProxy.getPort())
- && Objects.equals(proxy.getUsername(), otherProxy.getUsername())
- && Objects.equals(proxy.getPassword(), otherProxy.getPassword());
- }
-
- private static String normalize(String value) {
- return value == null ? null : value.toLowerCase(Locale.ROOT);
- }
}
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java b/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java
index 4d558955..37bfdcd2 100644
--- a/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java
@@ -17,6 +17,8 @@
package org.apache.stormcrawler.proxy;
+import java.util.Locale;
+import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -40,8 +42,8 @@ public class SCProxy {
private final String protocol;
private final String address;
private final String port;
- private String username;
- private String password;
+ private final String username;
+ private final String password;
private String country;
private String area;
private String location;
@@ -78,11 +80,15 @@ public class SCProxy {
this.port = matcher.group("port");
// load optional authentication data
+ String u = null;
+ String p = null;
try {
- this.username = matcher.group("username");
- this.password = matcher.group("password");
+ u = matcher.group("username");
+ p = matcher.group("password");
} catch (IllegalArgumentException ignored) {
}
+ this.username = u;
+ this.password = p;
}
/** Construct a proxy class from it's variables. */
@@ -106,12 +112,9 @@ public class SCProxy {
this.port = port;
// load optional parameters
- if (!username.isBlank()) {
- this.username = username;
- }
- if (!password.isBlank()) {
- this.password = password;
- }
+ this.username = username.isBlank() ? null : username;
+ this.password = password.isBlank() ? null : password;
+
if (!country.isBlank()) {
this.country = country;
}
@@ -186,4 +189,36 @@ public class SCProxy {
public String getStatus() {
return status;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (!(o instanceof SCProxy)) {
+ return false;
+ }
+
+ SCProxy other = (SCProxy) o;
+ return Objects.equals(normalize(this.protocol), normalize(other.protocol))
+ && Objects.equals(normalize(this.address), normalize(other.address))
+ && Objects.equals(this.port, other.port)
+ && Objects.equals(this.username, other.username)
+ && Objects.equals(this.password, other.password);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ normalize(this.protocol),
+ normalize(this.address),
+ this.port,
+ this.username,
+ this.password);
+ }
+
+ private static String normalize(String value) {
+ return value == null ? null : value.toLowerCase(Locale.ROOT);
+ }
}
diff --git a/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
index 1cf1e3f5..03ba55eb 100644
--- a/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
@@ -89,4 +89,75 @@ class SCProxyTest {
proxy.incrementUsage();
Assertions.assertEquals(1, proxy.getUsage());
}
+
+ @Test
+ void testEqualsAndHashCode() {
+ SCProxy a = new SCProxy("http://user1:pass1@example.com:8080");
+ SCProxy b = new SCProxy("http://user1:pass1@example.com:8080");
+
+ // equal on identical identity fields
+ Assertions.assertEquals(a, b);
+ // hashCode equal for equal objects
+ Assertions.assertEquals(a.hashCode(), b.hashCode());
+ }
+
+ @Test
+ void testEqualsCaseInsensitive() {
+ SCProxy a = new SCProxy("http://user1:pass1@example.com:8080");
+ SCProxy b = new SCProxy("HTTP://user1:pass1@EXAMPLE.COM:8080");
+
+ // protocol and address are case-insensitive
+ Assertions.assertEquals(a, b);
+ Assertions.assertEquals(a.hashCode(), b.hashCode());
+ }
+
+ @Test
+ void testNotEqualOnDifferentFields() {
+ SCProxy base = new SCProxy("http://user1:pass1@example.com:8080");
+
+ Assertions.assertNotEquals(
+ base, new SCProxy("http://user1:pass1@example.com:9090")); // different port
+ Assertions.assertNotEquals(
+ base, new SCProxy("http://user2:pass1@example.com:8080")); // different username
+ Assertions.assertNotEquals(
+ base, new SCProxy("http://user1:pass2@example.com:8080")); // different password
+ }
+
+ @Test
+ void testEqualsExcludesNonIdentityFields() {
+ // country, area, location, status are excluded from identity
+ SCProxy a =
+ new SCProxy(
+ "http",
+ "example.com",
+ "8080",
+ "user1",
+ "pass1",
+ "KR",
+ "Seoul",
+ "loc1",
+ "active");
+ SCProxy b =
+ new SCProxy(
+ "http",
+ "example.com",
+ "8080",
+ "user1",
+ "pass1",
+ "US",
+ "NY",
+ "loc2",
+ "inactive");
+
+ Assertions.assertEquals(a, b);
+ Assertions.assertEquals(a.hashCode(), b.hashCode());
+ }
+
+ @Test
+ void testEqualsNullAndDifferentType() {
+ SCProxy proxy = new SCProxy("http://user1:pass1@example.com:8080");
+
+ Assertions.assertNotEquals(proxy, null);
+ Assertions.assertNotEquals(proxy, "not-a-proxy");
+ }
}