You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ib...@apache.org on 2021/03/11 18:38:30 UTC
[beam] branch master updated: [BEAM-10961] Enable strict dependency
checking on Google Cloud Dataflow runner
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 13b38e1 [BEAM-10961] Enable strict dependency checking on Google Cloud Dataflow runner
new b675eb5 Merge pull request #13660 from sonam-vend/runner-google-cloud-dataflow
13b38e1 is described below
commit 13b38e12ea3a2becfeaa5bdecfe31280dc820034
Author: Sonam Ramchand <so...@venturedive.com>
AuthorDate: Tue Jan 5 18:50:24 2021 +0500
[BEAM-10961] Enable strict dependency checking on Google Cloud Dataflow runner
---
.../org/apache/beam/gradle/GrpcVendoring_1_26_0.groovy | 1 +
runners/google-cloud-dataflow-java/build.gradle | 8 ++++----
.../examples-streaming/build.gradle | 2 +-
.../google-cloud-dataflow-java/examples/build.gradle | 2 +-
runners/google-cloud-dataflow-java/worker/build.gradle | 14 ++++++++++----
.../worker/legacy-worker/build.gradle | 17 ++++++++++++-----
6 files changed, 29 insertions(+), 15 deletions(-)
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_26_0.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_26_0.groovy
index c2db11b..40e7383f 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_26_0.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_26_0.groovy
@@ -60,6 +60,7 @@ class GrpcVendoring_1_26_0 {
"io.netty:netty-transport-native-epoll:$netty_version",
// tcnative version from https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty
"io.netty:netty-tcnative-boringssl-static:2.0.33.Final",
+ "io.grpc:grpc-api:$grpc_version",
"com.google.auth:google-auth-library-credentials:$google_auth_version",
"io.grpc:grpc-testing:$grpc_version",
"com.google.api.grpc:proto-google-common-protos:$proto_google_common_protos_version",
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index 67f56f8..bbc1d4a 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -20,7 +20,6 @@ import groovy.json.JsonOutput
plugins { id 'org.apache.beam.module' }
applyJavaNature(
- enableStrictDependencies: false,
automaticModuleName: 'org.apache.beam.runners.dataflow',
classesTriggerCheckerBugs: [
'PrimitiveParDoSingleFactory': 'https://github.com/typetools/checker-framework/issues/3791',
@@ -68,6 +67,7 @@ configurations {
dependencies {
compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
+ permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
compile library.java.vendored_guava_26_0_jre
compile project(path: ":model:pipeline", configuration: "shadow")
compile project(path: ":sdks:java:core", configuration: "shadow")
@@ -80,17 +80,19 @@ dependencies {
compile library.java.google_api_services_clouddebugger
compile library.java.google_api_services_dataflow
compile library.java.google_api_services_storage
+ permitUnusedDeclared library.java.google_api_services_storage // BEAM-11761
compile library.java.google_auth_library_credentials
compile library.java.google_auth_library_oauth2_http
compile library.java.google_http_client
compile library.java.google_http_client_jackson2
+ permitUnusedDeclared library.java.google_http_client_jackson2 // BEAM-11761
+ compile library.java.hamcrest
compile library.java.jackson_annotations
compile library.java.jackson_core
compile library.java.jackson_databind
compile library.java.joda_time
compile library.java.slf4j_api
compile library.java.vendored_grpc_1_26_0
- testCompile library.java.hamcrest_core
testCompile library.java.guava_testlib
testCompile library.java.junit
testCompile project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntime")
@@ -100,8 +102,6 @@ dependencies {
testCompile library.java.google_cloud_dataflow_java_proto_library_all
testCompile library.java.jackson_dataformat_yaml
testCompile library.java.mockito_core
- testCompile library.java.proto_google_cloud_datastore_v1
- testCompile library.java.slf4j_jdk14
validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
validatesRunner project(project.path)
validatesRunner project(path: project.path, configuration: "testRuntime")
diff --git a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle
index 504a49b..fd8705d 100644
--- a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle
@@ -19,7 +19,7 @@
import groovy.json.JsonOutput
plugins { id 'org.apache.beam.module' }
-applyJavaNature(enableStrictDependencies: false, publish: false, exportJavadoc: false)
+applyJavaNature(publish: false, exportJavadoc: false)
// Evaluate the given project before this one, to allow referencing
// its sourceSets.test.output directly.
evaluationDependsOn(":examples:java")
diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle
index 891a72d..b128bc1 100644
--- a/runners/google-cloud-dataflow-java/examples/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples/build.gradle
@@ -19,7 +19,7 @@
import groovy.json.JsonOutput
plugins { id 'org.apache.beam.module' }
-applyJavaNature(enableStrictDependencies: false, publish: false, exportJavadoc: false)
+applyJavaNature(publish: false, exportJavadoc: false)
// Evaluate the given project before this one, to allow referencing
// its sourceSets.test.output directly.
evaluationDependsOn(":examples:java")
diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle b/runners/google-cloud-dataflow-java/worker/build.gradle
index f4a1877..a5156ab 100644
--- a/runners/google-cloud-dataflow-java/worker/build.gradle
+++ b/runners/google-cloud-dataflow-java/worker/build.gradle
@@ -27,7 +27,6 @@ def DATAFLOW_VERSION = "dataflow.version"
def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ? "com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION) : library.java.google_api_services_dataflow
applyJavaNature(
- enableStrictDependencies: false,
archivesBaseName: 'beam-runners-google-cloud-dataflow-java-fn-api-worker',
publish: false,
exportJavadoc: false,
@@ -71,6 +70,7 @@ dependencies {
// All main sourceset dependencies here should be listed as compile scope so that the dependencies
// are all packaged into a single uber jar allowing the jar to serve as an application.
compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
+ permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
compile project(":runners:google-cloud-dataflow-java")
compile project(path: ":sdks:java:core", configuration: "shadow")
compile project(":sdks:java:extensions:google-cloud-platform-core")
@@ -86,24 +86,32 @@ dependencies {
compile google_api_services_dataflow
compile library.java.avro
compile library.java.google_api_client
+ permitUnusedDeclared library.java.google_api_client // BEAM-11761
+ compile library.java.google_auth_library_credentials
compile library.java.google_http_client
compile library.java.google_http_client_jackson2
+ compile library.java.guava
compile library.java.jackson_annotations
compile library.java.jackson_core
compile library.java.jackson_databind
compile library.java.joda_time
+ compile library.java.proto_google_common_protos
shadow library.java.vendored_guava_26_0_jre
compile library.java.slf4j_api
+ compile "io.opencensus:opencensus-api:0.28.0"
compile "javax.servlet:javax.servlet-api:3.1.0"
// Conscrypt shouldn't be included here because Conscrypt won't work when being shaded.
// (Context: https://github.com/apache/beam/pull/13846)
// Conscrypt will be added to runtime dependencies by GrpcVendoring so compileOnly works for now.
- compileOnly "org.conscrypt:conscrypt-openjdk:2.5.1:linux-x86_64"
+ compileOnly "org.conscrypt:conscrypt-openjdk-uber:2.5.1"
+
compile "org.eclipse.jetty:jetty-server:9.2.10.v20150310"
compile "org.eclipse.jetty:jetty-servlet:9.2.10.v20150310"
compile library.java.error_prone_annotations
+ permitUnusedDeclared library.java.error_prone_annotations // BEAM-11761
compile library.java.slf4j_jdk14
+ permitUnusedDeclared library.java.slf4j_jdk14 // BEAM-11761
// All test sourceset dependencies can be marked as shadowTest since we create an uber jar without
// relocating any code.
@@ -112,8 +120,6 @@ dependencies {
shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntime")
shadowTest project(path: ":runners:direct-java", configuration: "shadow")
- shadowTest library.java.hamcrest_core
- shadowTest library.java.hamcrest_library
shadowTest library.java.jsonassert
shadowTest library.java.junit
shadowTest library.java.mockito_core
diff --git a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
index 1db7e37..fa193c0 100644
--- a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
+++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle
@@ -72,7 +72,6 @@ def excluded_dependencies = [
"com.google.auto.value:auto-value", // Provided scope added from applyJavaNature
"org.codehaus.jackson:jackson-core-asl", // Exclude an old version of jackson-core-asl introduced by google-http-client-jackson
"org.objenesis:objenesis", // Transitive dependency introduced from Beam
- "org.tukaani:xz", // Transitive dependency introduced from Beam
library.java.commons_compress, // Transitive dependency introduced from Beam
library.java.error_prone_annotations, // Provided scope added in worker
library.java.hamcrest_core, // Test only
@@ -82,7 +81,6 @@ def excluded_dependencies = [
]
applyJavaNature(
- enableStrictDependencies: false,
archivesBaseName: 'beam-runners-google-cloud-dataflow-java-legacy-worker',
publish: false,
classesTriggerCheckerBugs: [
@@ -201,6 +199,8 @@ dependencies {
}
compile project(path: ":model:fn-execution", configuration: "shadow")
+ compile project(path: ":model:pipeline", configuration: "shadow")
+ compile project(path: ":sdks:java:core", configuration: "shadow")
compile project(":runners:core-construction-java")
compile project(":runners:core-java")
compile project(":runners:java-fn-execution")
@@ -211,9 +211,18 @@ dependencies {
// Conscrypt shouldn't be included here because Conscrypt won't work when being shaded.
// (Context: https://github.com/apache/beam/pull/13846)
// Conscrypt will be added to runtime dependencies by GrpcVendoring so compileOnly works for now.
- compileOnly "org.conscrypt:conscrypt-openjdk:2.5.1:linux-x86_64"
+ compileOnly "org.conscrypt:conscrypt-openjdk-uber:2.5.1"
+
+ compile "javax.servlet:javax.servlet-api:3.1.0"
compile "org.eclipse.jetty:jetty-server:9.2.10.v20150310"
compile "org.eclipse.jetty:jetty-servlet:9.2.10.v20150310"
+ compile library.java.avro
+ compile library.java.jackson_annotations
+ compile library.java.jackson_core
+ compile library.java.jackson_databind
+ compile library.java.joda_time
+ compile library.java.slf4j_api
+ compile library.java.vendored_grpc_1_26_0
provided library.java.error_prone_annotations
runtime library.java.slf4j_jdk14
@@ -225,8 +234,6 @@ dependencies {
shadowTest project(path: ":runners:direct-java", configuration: "shadow")
shadowTest project(path: ":sdks:java:harness", configuration: "shadowTest")
shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
- shadowTest library.java.hamcrest_core
- shadowTest library.java.hamcrest_library
shadowTest library.java.jsonassert
shadowTest library.java.junit
shadowTest library.java.mockito_core