You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Hyukjin Kwon (Jira)" <ji...@apache.org> on 2022/05/02 02:06:00 UTC
[jira] [Commented] (SPARK-39066) Run `./dev/test-dependencies.sh --replace-manifest ` use Maven 3.8.5 produce wrong result
[ https://issues.apache.org/jira/browse/SPARK-39066?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17530584#comment-17530584 ]
Hyukjin Kwon commented on SPARK-39066:
--------------------------------------
You should better use `./build/mvn` to pin the version. we also use that for the release so the final jars won't be affected.
> Run `./dev/test-dependencies.sh --replace-manifest ` use Maven 3.8.5 produce wrong result
> -----------------------------------------------------------------------------------------
>
> Key: SPARK-39066
> URL: https://issues.apache.org/jira/browse/SPARK-39066
> Project: Spark
> Issue Type: Bug
> Components: Build
> Affects Versions: 3.4.0
> Reporter: Yang Jie
> Priority: Major
> Attachments: hadoop2-deps.txt, spark-deps-hadoop-2-hive-2.3
>
>
> Run `./dev/test-dependencies.sh --replace-manifest ` use maven 3.8.5 produce wrong result:
> {code:java}
> diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3
> index b6df3ea5ce..e803aadcfc 100644
> --- a/dev/deps/spark-deps-hadoop-2-hive-2.3
> +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3
> @@ -6,15 +6,14 @@ ST4/4.0.4//ST4-4.0.4.jar
> activation/1.1.1//activation-1.1.1.jar
> aircompressor/0.21//aircompressor-0.21.jar
> algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar
> +aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar
> +aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar
> +aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar
> +aliyun-sdk-oss/3.13.0//aliyun-sdk-oss-3.13.0.jar
> annotations/17.0.0//annotations-17.0.0.jar
> antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
> antlr4-runtime/4.8//antlr4-runtime-4.8.jar
> aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
> -aopalliance/1.0//aopalliance-1.0.jar
> -apacheds-i18n/2.0.0-M15//apacheds-i18n-2.0.0-M15.jar
> -apacheds-kerberos-codec/2.0.0-M15//apacheds-kerberos-codec-2.0.0-M15.jar
> -api-asn1-api/1.0.0-M20//api-asn1-api-1.0.0-M20.jar
> -api-util/1.0.0-M20//api-util-1.0.0-M20.jar
> arpack/2.2.1//arpack-2.2.1.jar
> arpack_combined_all/0.1//arpack_combined_all-0.1.jar
> arrow-format/7.0.0//arrow-format-7.0.0.jar
> @@ -26,7 +25,10 @@ automaton/1.11-8//automaton-1.11-8.jar
> avro-ipc/1.11.0//avro-ipc-1.11.0.jar
> avro-mapred/1.11.0//avro-mapred-1.11.0.jar
> avro/1.11.0//avro-1.11.0.jar
> -azure-storage/2.0.0//azure-storage-2.0.0.jar
> +aws-java-sdk-bundle/1.11.1026//aws-java-sdk-bundle-1.11.1026.jar
> +azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
> +azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
> +azure-storage/7.0.1//azure-storage-7.0.1.jar
> blas/2.2.1//blas-2.2.1.jar
> bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
> breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar
> @@ -34,28 +36,24 @@ breeze_2.12/1.2//breeze_2.12-1.2.jar
> cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar
> chill-java/0.10.0//chill-java-0.10.0.jar
> chill_2.12/0.10.0//chill_2.12-0.10.0.jar
> -commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
> commons-cli/1.5.0//commons-cli-1.5.0.jar
> commons-codec/1.15//commons-codec-1.15.jar
> commons-collections/3.2.2//commons-collections-3.2.2.jar
> commons-collections4/4.4//commons-collections4-4.4.jar
> commons-compiler/3.0.16//commons-compiler-3.0.16.jar
> commons-compress/1.21//commons-compress-1.21.jar
> -commons-configuration/1.6//commons-configuration-1.6.jar
> commons-crypto/1.1.0//commons-crypto-1.1.0.jar
> commons-dbcp/1.4//commons-dbcp-1.4.jar
> -commons-digester/1.8//commons-digester-1.8.jar
> -commons-httpclient/3.1//commons-httpclient-3.1.jar
> commons-io/2.4//commons-io-2.4.jar
> commons-lang/2.6//commons-lang-2.6.jar
> commons-lang3/3.12.0//commons-lang3-3.12.0.jar
> commons-logging/1.1.3//commons-logging-1.1.3.jar
> commons-math3/3.6.1//commons-math3-3.6.1.jar
> -commons-net/3.1//commons-net-3.1.jar
> commons-pool/1.5.4//commons-pool-1.5.4.jar
> commons-text/1.9//commons-text-1.9.jar
> compress-lzf/1.1//compress-lzf-1.1.jar
> core/1.1.2//core-1.1.2.jar
> +cos_api-bundle/5.6.19//cos_api-bundle-5.6.19.jar
> curator-client/2.7.1//curator-client-2.7.1.jar
> curator-framework/2.7.1//curator-framework-2.7.1.jar
> curator-recipes/2.7.1//curator-recipes-2.7.1.jar
> @@ -69,25 +67,17 @@ generex/1.0.2//generex-1.0.2.jar
> gmetric4j/1.0.10//gmetric4j-1.0.10.jar
> gson/2.2.4//gson-2.2.4.jar
> guava/14.0.1//guava-14.0.1.jar
> -guice-servlet/3.0//guice-servlet-3.0.jar
> -guice/3.0//guice-3.0.jar
> -hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar
> -hadoop-auth/2.7.4//hadoop-auth-2.7.4.jar
> -hadoop-aws/2.7.4//hadoop-aws-2.7.4.jar
> -hadoop-azure/2.7.4//hadoop-azure-2.7.4.jar
> -hadoop-client/2.7.4//hadoop-client-2.7.4.jar
> -hadoop-common/2.7.4//hadoop-common-2.7.4.jar
> -hadoop-hdfs/2.7.4//hadoop-hdfs-2.7.4.jar
> -hadoop-mapreduce-client-app/2.7.4//hadoop-mapreduce-client-app-2.7.4.jar
> -hadoop-mapreduce-client-common/2.7.4//hadoop-mapreduce-client-common-2.7.4.jar
> -hadoop-mapreduce-client-core/2.7.4//hadoop-mapreduce-client-core-2.7.4.jar
> -hadoop-mapreduce-client-jobclient/2.7.4//hadoop-mapreduce-client-jobclient-2.7.4.jar
> -hadoop-mapreduce-client-shuffle/2.7.4//hadoop-mapreduce-client-shuffle-2.7.4.jar
> -hadoop-openstack/2.7.4//hadoop-openstack-2.7.4.jar
> -hadoop-yarn-api/2.7.4//hadoop-yarn-api-2.7.4.jar
> -hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
> -hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
> -hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
> +hadoop-aliyun/3.3.2//hadoop-aliyun-3.3.2.jar
> +hadoop-annotations/3.3.2//hadoop-annotations-3.3.2.jar
> +hadoop-aws/3.3.2//hadoop-aws-3.3.2.jar
> +hadoop-azure-datalake/3.3.2//hadoop-azure-datalake-3.3.2.jar
> +hadoop-azure/3.3.2//hadoop-azure-3.3.2.jar
> +hadoop-client-api/3.3.2//hadoop-client-api-3.3.2.jar
> +hadoop-client-runtime/3.3.2//hadoop-client-runtime-3.3.2.jar
> +hadoop-cloud-storage/3.3.2//hadoop-cloud-storage-3.3.2.jar
> +hadoop-cos/3.3.2//hadoop-cos-3.3.2.jar
> +hadoop-openstack/3.3.2//hadoop-openstack-3.3.2.jar
> +hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar
> hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
> hive-beeline/2.3.9//hive-beeline-2.3.9.jar
> hive-cli/2.3.9//hive-cli-2.3.9.jar
> @@ -107,9 +97,9 @@ hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar
> hk2-api/2.6.1//hk2-api-2.6.1.jar
> hk2-locator/2.6.1//hk2-locator-2.6.1.jar
> hk2-utils/2.6.1//hk2-utils-2.6.1.jar
> -htrace-core/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
> httpclient/4.5.13//httpclient-4.5.13.jar
> httpcore/4.4.14//httpcore-4.4.14.jar
> +ini4j/0.5.4//ini4j-0.5.4.jar
> istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
> ivy/2.5.0//ivy-2.5.0.jar
> jackson-annotations/2.13.2//jackson-annotations-2.13.2.jar
> @@ -119,10 +109,8 @@ jackson-databind/2.13.2.1//jackson-databind-2.13.2.1.jar
> jackson-dataformat-cbor/2.13.2//jackson-dataformat-cbor-2.13.2.jar
> jackson-dataformat-yaml/2.13.2//jackson-dataformat-yaml-2.13.2.jar
> jackson-datatype-jsr310/2.13.2//jackson-datatype-jsr310-2.13.2.jar
> -jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
> jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
> jackson-module-scala_2.12/2.13.2//jackson-module-scala_2.12-2.13.2.jar
> -jackson-xc/1.9.13//jackson-xc-1.9.13.jar
> jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
> jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
> jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
> @@ -131,20 +119,21 @@ jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
> jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
> janino/3.0.16//janino-3.0.16.jar
> javassist/3.25.0-GA//javassist-3.25.0-GA.jar
> -javax.inject/1//javax.inject-1.jar
> javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
> javolution/5.5.1//javolution-5.5.1.jar
> jaxb-api/2.2.11//jaxb-api-2.2.11.jar
> jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
> jcl-over-slf4j/1.7.32//jcl-over-slf4j-1.7.32.jar
> jdo-api/3.0.1//jdo-api-3.0.1.jar
> +jdom2/2.0.6//jdom2-2.0.6.jar
> jersey-client/2.35//jersey-client-2.35.jar
> jersey-common/2.35//jersey-common-2.35.jar
> jersey-container-servlet-core/2.35//jersey-container-servlet-core-2.35.jar
> jersey-container-servlet/2.35//jersey-container-servlet-2.35.jar
> jersey-hk2/2.35//jersey-hk2-2.35.jar
> jersey-server/2.35//jersey-server-2.35.jar
> -jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar
> +jettison/1.1//jettison-1.1.jar
> +jetty-util-ajax/9.4.46.v20220331//jetty-util-ajax-9.4.46.v20220331.jar
> jetty-util/6.1.26//jetty-util-6.1.26.jar
> jetty-util/9.4.46.v20220331//jetty-util-9.4.46.v20220331.jar
> jetty/6.1.26//jetty-6.1.26.jar
> @@ -157,7 +146,6 @@ json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar
> json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar
> json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar
> json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar
> -jsp-api/2.1//jsp-api-2.1.jar
> jsr305/3.0.0//jsr305-3.0.0.jar
> jta/1.1//jta-1.1.jar
> jul-to-slf4j/1.7.32//jul-to-slf4j-1.7.32.jar
> @@ -219,6 +207,9 @@ objenesis/3.2//objenesis-3.2.jar
> okhttp/3.12.12//okhttp-3.12.12.jar
> okio/1.14.0//okio-1.14.0.jar
> opencsv/2.3//opencsv-2.3.jar
> +opentracing-api/0.33.0//opentracing-api-0.33.0.jar
> +opentracing-noop/0.33.0//opentracing-noop-0.33.0.jar
> +opentracing-util/0.33.0//opentracing-util-0.33.0.jar
> orc-core/1.7.4//orc-core-1.7.4.jar
> orc-mapreduce/1.7.4//orc-mapreduce-1.7.4.jar
> orc-shims/1.7.4//orc-shims-1.7.4.jar
> @@ -259,10 +250,8 @@ tink/1.6.1//tink-1.6.1.jar
> transaction-api/1.1//transaction-api-1.1.jar
> univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
> velocity/1.5//velocity-1.5.jar
> +wildfly-openssl/1.0.7.Final//wildfly-openssl-1.0.7.Final.jar
> xbean-asm9-shaded/4.20//xbean-asm9-shaded-4.20.jar
> -xercesImpl/2.12.0//xercesImpl-2.12.0.jar
> -xml-apis/1.4.01//xml-apis-1.4.01.jar
> -xmlenc/0.52//xmlenc-0.52.jar
> xz/1.8//xz-1.8.jar
> zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
> zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar
> {code}
> Executing `mvn dependency:tree -Phadoop-2 ` will also get wrong results,
> It seems that `-Phaoo` is not fully effective when using Maven 3.8.5, haven't found the root cause yet
>
>
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org