You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2020/06/02 22:03:32 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-1176] create
gobblin-all module resolving full dependency tree
This is an automated email from the ASF dual-hosted git repository.
hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new e9f35a8 [GOBBLIN-1176] create gobblin-all module resolving full dependency tree
e9f35a8 is described below
commit e9f35a8ce15275f843ec3920b7556c1f0d983c66
Author: vbohra <vb...@linkedin.com>
AuthorDate: Tue Jun 2 15:03:04 2020 -0700
[GOBBLIN-1176] create gobblin-all module resolving full dependency tree
Closes #3021 from vikrambohra/gobblinAll
---
gobblin-all/build.gradle | 98 ++++++++++++++++++++++
gobblin-rest-service/gobblin-rest-api/build.gradle | 4 +-
gobblin-restli/api.gradle | 8 +-
gradle/scripts/bintrayPublishing.gradle | 46 ++++++++++
settings.gradle | 1 +
5 files changed, 148 insertions(+), 9 deletions(-)
diff --git a/gobblin-all/build.gradle b/gobblin-all/build.gradle
new file mode 100644
index 0000000..cbdfc23
--- /dev/null
+++ b/gobblin-all/build.gradle
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'java'
+
+dependencies {
+ compile project(':gobblin-admin')
+ compile project(':gobblin-api')
+ compile project(':gobblin-audit')
+ compile project(':gobblin-modules:gobblin-avro-json')
+ compile project(':gobblin-aws')
+ compile project(':gobblin-modules:gobblin-azkaban')
+ compile project(':gobblin-modules:gobblin-azure-datalake')
+ compile project(':gobblin-binary-management')
+ compile project(':gobblin-cluster')
+ compile project(':gobblin-modules:gobblin-codecs')
+ compile project(':gobblin-compaction')
+ compile project(':gobblin-modules:gobblin-compliance')
+ compile project(':gobblin-config-management:gobblin-config-client')
+ compile project(':gobblin-config-management:gobblin-config-core')
+ compile project(':gobblin-config-management')
+ compile project(':gobblin-core')
+ compile project(':gobblin-core-base')
+ compile project(':gobblin-modules:gobblin-couchbase')
+ compile project(':gobblin-modules:gobblin-crypto')
+ compile project(':gobblin-modules:gobblin-crypto-provider')
+ compile project(':gobblin-data-management')
+ compile project(':gobblin-distribution')
+ compile project(':gobblin-modules:gobblin-elasticsearch')
+ compile project(':gobblin-modules:gobblin-elasticsearch-deps')
+ compile project(':gobblin-modules:gobblin-eventhub')
+ compile project(':gobblin-example')
+ compile project(':gobblin-restli:gobblin-flow-config-service')
+ compile project(':gobblin-restli:gobblin-flow-config-service:gobblin-flow-config-service-api')
+ compile project(path: ':gobblin-restli:gobblin-flow-config-service:gobblin-flow-config-service-api', configuration: 'dataTemplate')
+ compile project(path: ':gobblin-restli:gobblin-flow-config-service:gobblin-flow-config-service-api', configuration: 'restClient')
+ compile project(':gobblin-restli:gobblin-flow-config-service:gobblin-flow-config-service-client')
+ compile project(':gobblin-restli:gobblin-flow-config-service:gobblin-flow-config-service-server')
+ compile project(':gobblin-modules:gobblin-grok')
+ compile project(':gobblin-modules:gobblin-helix')
+ compile project(':gobblin-hive-registration')
+ compile project(':gobblin-modules:gobblin-http')
+ compile project(':gobblin-modules:gobblin-kafka-08')
+ compile project(':gobblin-modules:gobblin-kafka-common')
+ compile project(':gobblin-modules:gobblin-metadata')
+ compile project(':gobblin-metastore')
+ compile project(':gobblin-metrics-libs:gobblin-metrics')
+ compile project(':gobblin-metrics-libs:gobblin-metrics-base')
+ compile project(':gobblin-modules:gobblin-metrics-graphite')
+ compile project(':gobblin-modules:gobblin-metrics-hadoop')
+ compile project(':gobblin-modules:gobblin-metrics-influxdb')
+ compile project(':gobblin-metrics-libs')
+ compile project(':gobblin-modules:gobblin-orc-dep')
+ compile project(':gobblin-modules:gobblin-parquet')
+ compile project(':gobblin-modules:gobblin-parquet-apache')
+ compile project(':gobblin-modules:gobblin-parquet-common')
+ compile project(':gobblin-rest-service:gobblin-rest-api')
+ compile project(path: ':gobblin-rest-service:gobblin-rest-api', configuration: 'dataTemplate')
+ compile project(path: ':gobblin-rest-service:gobblin-rest-api', configuration: 'restClient')
+ compile project(':gobblin-rest-service:gobblin-rest-client')
+ compile project(':gobblin-rest-service:gobblin-rest-server')
+ compile project(':gobblin-rest-service')
+ compile project(':gobblin-restli')
+ compile project(':gobblin-restli:gobblin-restli-utils')
+ compile project(':gobblin-runtime')
+ compile project(':gobblin-runtime-hadoop')
+ compile project(':gobblin-salesforce')
+ compile project(':gobblin-service')
+ compile project(':gobblin-modules:gobblin-service-kafka')
+ compile project(':gobblin-modules:gobblin-sql')
+ compile project(':gobblin-test-harness')
+ compile project(':gobblin-test-utils')
+ compile project(':gobblin-restli:gobblin-throttling-service')
+ compile project(':gobblin-restli:gobblin-throttling-service:gobblin-throttling-service-api')
+ compile project(path: ':gobblin-restli:gobblin-throttling-service:gobblin-throttling-service-api', configuration: 'dataTemplate')
+ compile project(path: ':gobblin-restli:gobblin-throttling-service:gobblin-throttling-service-api', configuration: 'restClient')
+ compile project(':gobblin-restli:gobblin-throttling-service:gobblin-throttling-service-client')
+ compile project(':gobblin-restli:gobblin-throttling-service:gobblin-throttling-service-server')
+ compile project(':gobblin-tunnel')
+ compile project(':gobblin-utility')
+ compile project(':gobblin-yarn')
+ compile project(':gobblin-modules:gobblin-zuora')
+ compile project(':gobblin-modules:google-ingestion')
+}
\ No newline at end of file
diff --git a/gobblin-rest-service/gobblin-rest-api/build.gradle b/gobblin-rest-service/gobblin-rest-api/build.gradle
index 5318611..e792a26 100644
--- a/gobblin-rest-service/gobblin-rest-api/build.gradle
+++ b/gobblin-rest-service/gobblin-rest-api/build.gradle
@@ -112,8 +112,7 @@ artifacts {
archives dataTemplateSourcesJar, dataTemplateJavadocJar, restClientSourcesJar, restClientJavadocJar
}
-publishing {
- publications {
+publishing.publications {
dataTemplate(MavenPublication) {
artifactId dataTemplateName
artifacts = [mainDataTemplateJar, dataTemplateSourcesJar, dataTemplateJavadocJar]
@@ -126,7 +125,6 @@ publishing {
pom pomAttributes
pom.withXml addRuntimeDependenciesToPom
}
- }
}
addPublicationToBintray("dataTemplate")
addPublicationToBintray("restClient")
diff --git a/gobblin-restli/api.gradle b/gobblin-restli/api.gradle
index f1b5ab4..d58c419 100644
--- a/gobblin-restli/api.gradle
+++ b/gobblin-restli/api.gradle
@@ -95,15 +95,13 @@ if (file("${project.projectDir}/src/main/idl").exists()) {
archives mainRestClientJar, restClientSourcesJar, restClientJavadocJar
}
- publishing {
- publications {
+ publishing.publications {
restClient(MavenPublication) {
artifactId restClientName
artifacts = [mainRestClientJar, restClientSourcesJar, restClientJavadocJar]
pom pomAttributes
pom.withXml addRuntimeDependenciesToPom
}
- }
}
addPublicationToBintray("restClient")
}
@@ -136,14 +134,12 @@ if (rootProject.ext.publishToMaven || rootProject.ext.publishToNexus) {
}
}
-publishing {
- publications {
+publishing.publications {
dataTemplate(MavenPublication) {
artifactId dataTemplateName
artifacts = [mainDataTemplateJar, dataTemplateSourcesJar, dataTemplateJavadocJar]
pom pomAttributes
pom.withXml addRuntimeDependenciesToPom
}
- }
}
addPublicationToBintray("dataTemplate")
diff --git a/gradle/scripts/bintrayPublishing.gradle b/gradle/scripts/bintrayPublishing.gradle
index ea435f3..902727f 100644
--- a/gradle/scripts/bintrayPublishing.gradle
+++ b/gradle/scripts/bintrayPublishing.gradle
@@ -52,6 +52,52 @@ subprojects{
artifact javadocJar
pom pomAttributes
+
+ pom.withXml {
+ //Ensures that correct dependencies are in the pom when subproject declares a project dependency
+ // with specific target configuration (restClient, dataTemplate, etc.)
+ //Needed because pom model is lossy and does not carry the 'configuration' information
+ def dependenciesNode = it.asNode().dependencies[0]
+ def removed = [] as Set; def added = [] //helps auditing
+ configurations.runtime.allDependencies.each { d ->
+ def confToPom = ['restClient': 'rest-client', 'dataTemplate': 'data-template']
+ if (d instanceof ProjectDependency && confToPom.containsKey(d.targetConfiguration)) {
+ boolean dependsOnMainModule = configurations.runtime.allDependencies.any {
+ it.name == d.name && it.targetConfiguration == 'default'
+ }
+ if (!dependsOnMainModule) {
+ //subproject declares a dependency on target configuration (i.e. path: 'gobblin-rest-api', configuration: 'restClient')
+ // but does not declare a dependency on the 'default' artifact (i.e. 'gobblin-rest-api')
+ //we need to remove the 'default' artifact from the pom file
+ def mainModuleNode = dependenciesNode.find { it.artifactId.text() == d.name }
+ dependenciesNode.remove(mainModuleNode)
+ removed.add(d.name)
+ }
+
+ //adding explicit dependency on the artifact that corresponds to given target configuration
+ // (i.e. 'gobblin-rest-api-rest-client')
+ def dependencyNode = dependenciesNode.appendNode('dependency')
+ dependencyNode.appendNode('groupId', d.group)
+ def newArtifactId = d.name + "-" + confToPom[d.targetConfiguration]
+ dependencyNode.appendNode('artifactId', newArtifactId)
+ dependencyNode.appendNode('version', d.version)
+ added.add(newArtifactId)
+
+ //adding global exclude rules to the new dependency entries
+ if (configurations.runtime.excludeRules.size() > 0) {
+ def exclusionsNode = dependencyNode.appendNode('exclusions')
+ configurations.runtime.excludeRules.each { rule ->
+ def exclusionNode = exclusionsNode.appendNode('exclusion')
+ exclusionNode.appendNode('groupId', rule.group ?: "*")
+ exclusionNode.appendNode('artifactId', rule.module)
+ }
+ }
+ }
+ }
+ if (added || removed) {
+ logger.lifecycle("Updated pom dependencies in {}, added: {}, removed: {}", project.path, added, removed)
+ }
+ }
}
}
}
diff --git a/settings.gradle b/settings.gradle
index f1ceb05..952e628 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -16,6 +16,7 @@
*/
def modules = ['gobblin-admin',
+ 'gobblin-all',
'gobblin-api',
'gobblin-compaction',
'gobblin-config-management',