You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2019/06/11 18:03:03 UTC
[kudu] 03/03: [build] Fix jar verification via verify_jars.pl
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 988296dcd1e2d5876f41ca2d481b126523f5a457
Author: Grant Henke <gr...@apache.org>
AuthorDate: Tue Jun 11 12:05:29 2019 -0500
[build] Fix jar verification via verify_jars.pl
This patch fixes jar verification with a few changes.
It fixes some dependencies in kudu-backup and
kudu-test-utils to be sure we aren’t pulling in
unwanted jars. It also updates verify_jars.pl to allow
various relocated classes and the Spark
DataSourceRegister file. It also ignores the buildSrc,
kudu-jepsen and kudu-backup-tools jars.
Change-Id: I57272ff8c1db043c7be00f7f65c11d010b33c8b5
Reviewed-on: http://gerrit.cloudera.org:8080/13584
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Reviewed-by: Mike Percy <mp...@apache.org>
Tested-by: Grant Henke <gr...@apache.org>
---
build-support/verify_jars.pl | 17 ++++++++++++++---
java/gradle/dependencies.gradle | 3 +--
java/gradle/shadow.gradle | 9 +++++++++
java/kudu-backup/build.gradle | 2 ++
java/kudu-client/build.gradle | 2 --
java/kudu-test-utils/build.gradle | 6 +++++-
6 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/build-support/verify_jars.pl b/build-support/verify_jars.pl
index 7b93ee3..4209c44 100755
--- a/build-support/verify_jars.pl
+++ b/build-support/verify_jars.pl
@@ -29,13 +29,19 @@ my $pat_kudu_shaded_prefix = qr{^org/apache/kudu/shaded/};
# Allowed filenames of non-Java files in JARs.
my $pat_allow_non_java =
qr{(?:\.(?:txt|xml|properties|proto|MF)|
- LICENSE|NOTICE|DEPENDENCIES)$}x;
+ LICENSE|NOTICE|DEPENDENCIES|
+ # The kudu-spark DataSourceRegister file.
+ DataSourceRegister)$}x;
# Allowed filenames of shaded dependencies in JARs.
my $pat_allow_kudu_shaded =
qr{^org/apache/kudu/shaded/
- (?:com/google/(?:common|gson|protobuf|thirdparty/publicsuffix)|
+ (?:com/google/(?:common|gson|gradle/osdetector|protobuf|thirdparty/publicsuffix)|
com/sangupta/murmur|
+ kr/motd/maven|
+ org/apache/(?:commons|http)|
+ org/checkerframework|
+ org/HdrHistogram|
org/jboss/netty|
scopt)
}x;
@@ -67,7 +73,12 @@ chomp(my @jars = `find . -type f -name \*.jar |
grep -v sources\.jar |
grep -v javadoc\.jar |
grep -v unshaded\.jar |
- grep -v gradle-wrapper.jar`);
+ grep -v buildSrc.jar |
+ grep -v gradle-wrapper.jar |
+ # Ignored because it's test only and unpublished.
+ grep -v kudu-jepsen.*\.jar |
+ # Ignored because it's a tool jar that shades everything.
+ grep -v kudu-backup-tools.*\.jar`);
my $num_errors = 0;
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index 6721564..c80763e 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -87,8 +87,7 @@ libs += [
hadoopCommon : "org.apache.hadoop:hadoop-common:$versions.hadoop",
hadoopMRClientCommon : "org.apache.hadoop:hadoop-mapreduce-client-common:$versions.hadoop",
hadoopMRClientCore : "org.apache.hadoop:hadoop-mapreduce-client-core:$versions.hadoop",
- hamcrestCore : "org.hamcrest:hamcrest-core:$versions.hamcrest",
- hamcrestLib : "org.hamcrest:hamcrest-library:$versions.hamcrest",
+ hamcrest : "org.hamcrest:hamcrest:$versions.hamcrest",
hdrhistogram : "org.hdrhistogram:HdrHistogram:$versions.hdrhistogram",
hiveMetastore : "org.apache.hive:hive-metastore:$versions.hive",
hiveMetastoreTest : "org.apache.hive:hive-metastore:$versions.hive:tests",
diff --git a/java/gradle/shadow.gradle b/java/gradle/shadow.gradle
index 499554f..a46e678 100644
--- a/java/gradle/shadow.gradle
+++ b/java/gradle/shadow.gradle
@@ -45,10 +45,19 @@ configurations.archives.artifacts.removeAll {
// location across all modules.
shadowJar {
relocate "com.google.common", "org.apache.kudu.shaded.com.google.common"
+ relocate "com.google.gradle.osdetector", "org.apache.kudu.shaded.com.google.gradle.osdetector"
relocate "com.google.gson", "org.apache.kudu.shaded.com.google.gson"
relocate "com.google.protobuf", "org.apache.kudu.shaded.com.google.protobuf"
relocate "com.google.thirdparty", "org.apache.kudu.shaded.com.google.thirdparty"
relocate "com.sangupta", "org.apache.kudu.shaded.com.sangupta"
+ // Pulled in via osdetector.
+ relocate "kr.motd.maven", "org.apache.kudu.shaded.kr.motd.maven"
+ relocate "org.apache.http", "org.apache.kudu.shaded.org.apache.http"
+ relocate "org.apache.commons", "org.apache.kudu.shaded.org.apache.commons"
+ // Pulled in via Guava.
+ relocate "org.checkerframework", "org.apache.kudu.shaded.org.checkerframework"
+ relocate "org.hamcrest", "org.apache.kudu.shaded.org.hamcrest"
+ relocate "org.HdrHistogram", "org.apache.kudu.shaded.org.HdrHistogram"
relocate "org.jboss.netty", "org.apache.kudu.shaded.org.jboss.netty"
relocate "scopt", "org.apache.kudu.shaded.scopt"
}
diff --git a/java/kudu-backup/build.gradle b/java/kudu-backup/build.gradle
index 47d414e..7ee49f5 100644
--- a/java/kudu-backup/build.gradle
+++ b/java/kudu-backup/build.gradle
@@ -23,6 +23,8 @@ dependencies {
compile(project(path: ":kudu-backup-common")) {
// Ensure we use the hadoop-client provided by Spark to avoid any compatibility issues.
exclude group: "org.apache.hadoop", module: "hadoop-common"
+ // Ensure we use the scala-library provided by Spark to avoid any compatibility issues.
+ exclude group: "org.scala-lang", module: "scala-library"
}
compile project(path: ":kudu-client", configuration: "shadow")
compile project(path: ":kudu-spark", configuration: "shadow")
diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle
index e42b4eb..6ac2f07 100644
--- a/java/kudu-client/build.gradle
+++ b/java/kudu-client/build.gradle
@@ -36,8 +36,6 @@ dependencies {
optional libs.yetusAnnotations
testCompile project(":kudu-test-utils")
- testCompile libs.hamcrestCore
- testCompile libs.hamcrestLib
testCompile libs.hiveMetastore
// The HMS client relies on the MR client-core artifact for JobConf, but only
// specifies it as an optional dependency. Gradle doesn't pull in optional
diff --git a/java/kudu-test-utils/build.gradle b/java/kudu-test-utils/build.gradle
index 3e03012..72db57c 100644
--- a/java/kudu-test-utils/build.gradle
+++ b/java/kudu-test-utils/build.gradle
@@ -25,7 +25,11 @@ dependencies {
compile libs.httpMime
compile libs.osdetector
- compileUnshaded libs.junit
+ compileUnshaded libs.hamcrest
+ compileUnshaded(libs.junit) {
+ // Use the Kudu specified hamcrest.
+ exclude group: "org.hamcrest"
+ }
compileUnshaded libs.slf4jApi
// Support using any kudu-binary jar for tests via `-PuseBinJar=<version>`.