You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2024/07/23 11:01:44 UTC

(incubator-stormcrawler) 01/01: #1251 - Update to Storm 2.6.3

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch storm-2.6.3
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git

commit 2acc49efeb163afa4ad44e74c2693a2c1d5b9ad5
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Tue Jul 23 13:01:36 2024 +0200

    #1251 - Update to Storm 2.6.3
---
 THIRD-PARTY.txt       | 47 ++++++++++++++++-------------------------------
 external/warc/pom.xml | 12 ------------
 pom.xml               |  2 +-
 3 files changed, 17 insertions(+), 44 deletions(-)

diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt
index a05648bb..0abe2ad3 100644
--- a/THIRD-PARTY.txt
+++ b/THIRD-PARTY.txt
@@ -8,6 +8,10 @@ List of third-party dependencies grouped by their license type.
         * MinLog (com.esotericsoftware:minlog:1.3.1 - https://github.com/EsotericSoftware/minlog)
         * ReflectASM (com.esotericsoftware:reflectasm:1.11.9 - https://github.com/EsotericSoftware/reflectasm)
 
+    AL 2.0, GPL v2, MPL 2.0
+
+        * RabbitMQ Java Client (com.rabbitmq:amqp-client:5.21.0 - https://www.rabbitmq.com)
+
     Apache License
 
         * Log4j Implemented Over SLF4J (org.slf4j:log4j-over-slf4j:1.7.36 - http://www.slf4j.org)
@@ -20,25 +24,20 @@ List of third-party dependencies grouped by their license type.
         * Apache Commons Codec (commons-codec:commons-codec:1.11 - http://commons.apache.org/proper/commons-codec/)
         * Apache Commons Codec (commons-codec:commons-codec:1.15 - https://commons.apache.org/proper/commons-codec/)
         * Apache Commons Codec (commons-codec:commons-codec:1.16.0 - https://commons.apache.org/proper/commons-codec/)
-        * Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/)
         * Apache Commons Collections (org.apache.commons:commons-collections4:4.4 - https://commons.apache.org/proper/commons-collections/)
         * Apache Commons Compress (org.apache.commons:commons-compress:1.22 - https://commons.apache.org/proper/commons-compress/)
         * Apache Commons Compress (org.apache.commons:commons-compress:1.24.0 - https://commons.apache.org/proper/commons-compress/)
-        * Apache Commons Configuration (org.apache.commons:commons-configuration2:2.10.1 - https://commons.apache.org/proper/commons-configuration/)
         * Apache Commons Crypto (org.apache.commons:commons-crypto:1.1.0 - https://commons.apache.org/proper/commons-crypto/)
         * Apache Commons CSV (org.apache.commons:commons-csv:1.10.0 - https://commons.apache.org/proper/commons-csv/)
         * Apache Commons Exec (org.apache.commons:commons-exec:1.3 - http://commons.apache.org/proper/commons-exec/)
         * Apache Commons IO (commons-io:commons-io:2.14.0 - https://commons.apache.org/proper/commons-io/)
         * Apache Commons Lang (org.apache.commons:commons-lang3:3.13.0 - https://commons.apache.org/proper/commons-lang/)
-        * Apache Commons Lang (org.apache.commons:commons-lang3:3.14.0 - https://commons.apache.org/proper/commons-lang/)
+        * Apache Commons Lang (org.apache.commons:commons-lang3:3.9 - http://commons.apache.org/proper/commons-lang/)
         * Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/)
         * Apache Commons Math (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/)
-        * Apache Commons Text (org.apache.commons:commons-text:1.11.0 - https://commons.apache.org/proper/commons-text)
         * Apache FontBox (org.apache.pdfbox:fontbox:2.0.29 - http://pdfbox.apache.org/)
-        * Apache Hadoop Auth (org.apache.hadoop:hadoop-auth:3.3.6 - no url defined)
-        * Apache Hadoop Client API (org.apache.hadoop:hadoop-client-api:3.3.6 - no url defined)
-        * Apache Hadoop Common (org.apache.hadoop:hadoop-common:3.3.6 - no url defined)
-        * Apache Hadoop shaded Guava (org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 - https://www.apache.org/hadoop-thirdparty/hadoop-shaded-guava/)
+        * Apache Hadoop Client API (org.apache.hadoop:hadoop-client-api:3.4.0 - no url defined)
+        * Apache Hadoop Client Runtime (org.apache.hadoop:hadoop-client-runtime:3.4.0 - no url defined)
         * Apache HBase - Client (org.apache.hbase:hbase-client:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-client)
         * Apache HBase - Common (org.apache.hbase:hbase-common:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-common)
         * Apache HBase - Hadoop Compatibility (org.apache.hbase:hbase-hadoop-compat:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-hadoop-compat)
@@ -135,7 +134,7 @@ List of third-party dependencies grouped by their license type.
         * Failsafe (dev.failsafe:failsafe:3.3.2 - https://failsafe.dev/failsafe)
         * FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/)
         * Google Android Annotations Library (com.google.android:annotations:4.1.1.4 - http://source.android.com/)
-        * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:3.2.6 - http://metrics.dropwizard.io/metrics-graphite/)
+        * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:4.2.26 - https://metrics.dropwizard.io/metrics-graphite)
         * Gson (com.google.code.gson:gson:2.10.1 - https://github.com/google/gson/gson)
         * Gson (com.google.code.gson:gson:2.9.0 - https://github.com/google/gson/gson)
         * Guava: Google Core Libraries for Java (com.google.guava:guava:18.0 - http://code.google.com/p/guava-libraries/guava)
@@ -169,7 +168,6 @@ List of third-party dependencies grouped by their license type.
         * Jackson dataformat: Smile (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.16.1 - https://github.com/FasterXML/jackson-dataformats-binary)
         * Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.16.1 - https://github.com/FasterXML/jackson-dataformats-text)
         * java-libpst (com.pff:java-libpst:0.9.3 - https://github.com/rjohnsondev/java-libpst)
-        * JCIP Annotations under Apache License (com.github.stephenc.jcip:jcip-annotations:1.0-1 - http://stephenc.github.com/jcip-annotations)
         * JCL 1.2 implemented over SLF4J (org.slf4j:jcl-over-slf4j:2.0.10 - http://www.slf4j.org)
         * JCL 1.2 implemented over SLF4J (org.slf4j:jcl-over-slf4j:2.0.9 - http://www.slf4j.org)
         * JetBrains Java Annotations (org.jetbrains:annotations:24.1.0 - https://github.com/JetBrains/java-annotations)
@@ -177,19 +175,9 @@ List of third-party dependencies grouped by their license type.
         * Joda-Time (joda-time:joda-time:2.12.2 - https://www.joda.org/joda-time/)
         * Joda-Time (joda-time:joda-time:2.8.1 - http://www.joda.org/joda-time/)
         * jsonic (net.arnx:jsonic:1.2.11 - http://jsonic.sourceforge.jp/)
-        * JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:3.2.6 - http://metrics.dropwizard.io/metrics-jvm/)
+        * JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:4.2.26 - https://metrics.dropwizard.io/metrics-jvm)
         * jwarc (org.netpreserve:jwarc:0.28.3 - https://github.com/iipc/jwarc)
         * jwarc (org.netpreserve:jwarc:0.29.0 - https://github.com/iipc/jwarc)
-        * Kerb Simple Kdc (org.apache.kerby:kerb-simplekdc:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-simplekdc)
-        * Kerby ASN1 Project (org.apache.kerby:kerby-asn1:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-asn1)
-        * Kerby Config (org.apache.kerby:kerby-config:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-config)
-        * Kerby-kerb Client (org.apache.kerby:kerb-client:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-client)
-        * Kerby-kerb Common (org.apache.kerby:kerb-common:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-common)
-        * Kerby-kerb core (org.apache.kerby:kerb-core:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-core)
-        * Kerby-kerb Crypto (org.apache.kerby:kerb-crypto:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-crypto)
-        * Kerby-kerb Util (org.apache.kerby:kerb-util:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-util)
-        * Kerby PKIX Project (org.apache.kerby:kerby-pkix:1.0.1 - http://directory.apache.org/kerby/kerby-pkix)
-        * Kerby Util (org.apache.kerby:kerby-util:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-util)
         * Kotlin Stdlib (org.jetbrains.kotlin:kotlin-stdlib:1.8.21 - https://kotlinlang.org/)
         * Kotlin Stdlib Common (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 - https://kotlinlang.org/)
         * Kotlin Stdlib Jdk7 (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.8.21 - https://kotlinlang.org/)
@@ -197,7 +185,8 @@ List of third-party dependencies grouped by their license type.
         * lang-mustache (org.opensearch.plugin:lang-mustache-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
         * language-detector (com.optimaize.languagedetector:language-detector:0.6 - https://github.com/optimaize/language-detector)
         * mapper-extras (org.opensearch.plugin:mapper-extras-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
-        * Metrics Core (io.dropwizard.metrics:metrics-core:3.2.6 - http://metrics.dropwizard.io/metrics-core/)
+        * Metrics Core (io.dropwizard.metrics:metrics-core:4.2.26 - https://metrics.dropwizard.io/metrics-core)
+        * Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.26 - https://metrics.dropwizard.io/metrics-jmx)
         * Netty/Buffer (io.netty:netty-buffer:4.1.94.Final - https://netty.io/netty-buffer/)
         * Netty/Codec (io.netty:netty-codec:4.1.94.Final - https://netty.io/netty-codec/)
         * Netty/Common (io.netty:netty-common:4.1.94.Final - https://netty.io/netty-common/)
@@ -209,7 +198,6 @@ List of third-party dependencies grouped by their license type.
         * Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.94.Final - https://netty.io/netty-transport-native-epoll/)
         * Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.94.Final - https://netty.io/netty-transport-native-unix-common/)
         * Netty/Transport (io.netty:netty-transport:4.1.94.Final - https://netty.io/netty-transport/)
-        * Nimbus JOSE+JWT (com.nimbusds:nimbus-jose-jwt:9.8.1 - https://bitbucket.org/connect2id/nimbus-jose-jwt)
         * Non-Blocking Reactive Foundation for the JVM (io.projectreactor:reactor-core:3.5.14 - https://github.com/reactor/reactor-core)
         * Objenesis (org.objenesis:objenesis:3.3 - http://objenesis.org/objenesis)
         * Ogg and Vorbis for Java, Core (org.gagravarr:vorbis-java-core:0.8 - https://github.com/Gagravarr/VorbisJava)
@@ -261,20 +249,18 @@ List of third-party dependencies grouped by their license type.
         * rome (com.rometools:rome:2.1.0 - http://rometools.com/rome)
         * rome-utils (com.rometools:rome-utils:2.1.0 - http://rometools.com/rome-utils)
         * server (org.opensearch:opensearch:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
-        * Shaded Deps for Storm Client (org.apache.storm:storm-shaded-deps:2.6.2 - https://storm.apache.org/storm-shaded-deps)
+        * Shaded Deps for Storm Client (org.apache.storm:storm-shaded-deps:2.6.3 - https://storm.apache.org/storm-shaded-deps)
         * SnakeYAML (org.yaml:snakeyaml:2.2 - https://bitbucket.org/snakeyaml/snakeyaml)
-        * snappy-java (org.xerial.snappy:snappy-java:1.1.8.2 - https://github.com/xerial/snappy-java)
+        * snappy-java (org.xerial.snappy:snappy-java:1.1.10.4 - https://github.com/xerial/snappy-java)
         * sniffer (org.opensearch.client:opensearch-rest-client-sniffer:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
         * SparseBitSet (com.zaxxer:SparseBitSet:1.2 - https://github.com/brettwooldridge/SparseBitSet)
-        * storm-autocreds (org.apache.storm:storm-autocreds:2.6.2 - https://storm.apache.org/storm-autocreds)
-        * Storm Client (org.apache.storm:storm-client:2.6.2 - https://storm.apache.org/storm-client)
-        * storm-hdfs (org.apache.storm:storm-hdfs:2.6.2 - https://storm.apache.org/storm-hdfs)
+        * storm-autocreds (org.apache.storm:storm-autocreds:2.6.3 - https://storm.apache.org/storm-autocreds)
+        * Storm Client (org.apache.storm:storm-client:2.6.3 - https://storm.apache.org/storm-client)
+        * storm-hdfs (org.apache.storm:storm-hdfs:2.6.3 - https://storm.apache.org/storm-hdfs)
         * swagger-annotations-jakarta (io.swagger.core.v3:swagger-annotations-jakarta:2.2.17 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations-jakarta)
         * TagSoup (org.ccil.cowan.tagsoup:tagsoup:1.2.1 - http://home.ccil.org/~cowan/XML/tagsoup/)
         * T-Digest (com.tdunning:t-digest:3.2 - https://github.com/tdunning/t-digest)
-        * Token provider (org.apache.kerby:token-provider:1.0.1 - http://directory.apache.org/kerby/kerby-provider/token-provider)
         * urlfrontier-API (com.github.crawler-commons:urlfrontier-API:2.3.1 - https://github.com/crawler-commons/url-frontier/urlfrontier-API)
-        * Woodstox (com.fasterxml.woodstox:woodstox-core:5.4.0 - https://github.com/FasterXML/woodstox)
         * Xerces2-j (xerces:xercesImpl:2.12.2 - https://xerces.apache.org/xerces2-j/)
         * XmlBeans (org.apache.xmlbeans:xmlbeans:5.1.1 - https://xmlbeans.apache.org/)
 
@@ -327,7 +313,6 @@ List of third-party dependencies grouped by their license type.
         * curvesapi (com.github.virtuald:curvesapi:1.07 - https://github.com/virtuald/curvesapi)
         * JMatIO (org.tallison:jmatio:1.5 - https://github.com/tballison/jmatio)
         * JZlib (com.jcraft:jzlib:1.1.3 - http://www.jcraft.com/jzlib/)
-        * Stax2 API (org.codehaus.woodstox:stax2-api:4.2 - http://github.com/FasterXML/stax2-api)
 
     CDDL, v1.0, LGPL, v2.1 or later
 
diff --git a/external/warc/pom.xml b/external/warc/pom.xml
index ae0b953b..98f46ef9 100644
--- a/external/warc/pom.xml
+++ b/external/warc/pom.xml
@@ -73,21 +73,9 @@ under the License.
 					<groupId>org.apache.hive.hcatalog</groupId>
 					<artifactId>hive-hcatalog-streaming</artifactId>
 				</exclusion>
-				<!-- this is only needed if explicitly configured and will be dropped in 2.6.2 -->
-				<exclusion>
-					<groupId>io.confluent</groupId>
-					<artifactId>kafka-avro-serializer</artifactId>
-				</exclusion>
 			</exclusions>
 		</dependency>
 
-		<!-- https://github.com/apache/incubator-stormcrawler/pull/1189 -->
-		<dependency>
-			<groupId>org.apache.hadoop</groupId>
-			<artifactId>hadoop-client-api</artifactId>
-			<version>3.3.6</version>
-		</dependency>
-
 		<dependency>
 			<groupId>org.netpreserve</groupId>
 			<artifactId>jwarc</artifactId>
diff --git a/pom.xml b/pom.xml
index 0025de0f..9473c86d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,7 +63,7 @@ under the License.
 		<additionalparam>-Xdoclint:none</additionalparam>
 		<!-- dependency versions -->
 		<junit.version>5.10.2</junit.version>
-		<storm-client.version>2.6.2</storm-client.version>
+		<storm-client.version>2.6.3</storm-client.version>
 		<jackson.version>2.15.2</jackson.version>
 		<tika.version>2.9.1</tika.version>
 		<mockito.version>5.10.0</mockito.version>