You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/08/25 03:03:16 UTC
[1/3] kudu git commit: [build] Fix build jar verification
Repository: kudu
Updated Branches:
refs/heads/master 264b5b20f -> be360bf01
[build] Fix build jar verification
- Adjusts the verify_jars.pl script to ignore jars in
build/jars path. This is where all the dependency jars
are staged for dist-tests. They are not included in the
distribution.
- Shades gson and scopt in kudu-backup and adjusts
verify_jars.pl to allow it to be relocated when shading.
- Adjust the kudu-backup pom to relocate the
needed classes.
Change-Id: Ib3789b6ec379bf715499d1142576218ed7344258
Reviewed-on: http://gerrit.cloudera.org:8080/11318
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Grant Henke <gr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/245f0a7e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/245f0a7e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/245f0a7e
Branch: refs/heads/master
Commit: 245f0a7e0d412c9a820746987f64c034a710ef1e
Parents: 264b5b2
Author: Grant Henke <gr...@apache.org>
Authored: Fri Aug 24 15:52:27 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Sat Aug 25 02:33:20 2018 +0000
----------------------------------------------------------------------
build-support/verify_jars.pl | 11 +++++++----
java/gradle/shadow.gradle | 2 ++
java/kudu-backup/pom.xml | 37 +++++++++++++++++++++++++++++++++++++
3 files changed, 46 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/245f0a7e/build-support/verify_jars.pl
----------------------------------------------------------------------
diff --git a/build-support/verify_jars.pl b/build-support/verify_jars.pl
index 45352a8..7b93ee3 100755
--- a/build-support/verify_jars.pl
+++ b/build-support/verify_jars.pl
@@ -34,8 +34,10 @@ my $pat_allow_non_java =
# Allowed filenames of shaded dependencies in JARs.
my $pat_allow_kudu_shaded =
qr{^org/apache/kudu/shaded/
- (?:org/jboss/netty|com/google/(?:common|protobuf|thirdparty/publicsuffix)|
- com/sangupta/murmur)
+ (?:com/google/(?:common|gson|protobuf|thirdparty/publicsuffix)|
+ com/sangupta/murmur|
+ org/jboss/netty|
+ scopt)
}x;
# Allowed paths of unshaded Kudu dependencies in JARs.
@@ -44,9 +46,9 @@ my $pat_allow_kudu_unshaded = qr{^org/apache/kudu/.*};
# Allowed paths of unshaded non-Kudu dependencies in JARs.
my $pat_allow_nonkudu_unshaded = qr{^(?:com/databricks/spark/avro|
+ com/stumbleupon/async/|
org/apache/parquet/|
- org/apache/yetus/|
- com/stumbleupon/async/)}x;
+ org/apache/yetus/)}x;
if (scalar @ARGV != 1) {
print STDERR "Usage: $0 <dest_dir>\n";
@@ -58,6 +60,7 @@ chdir($dest_dir) or die "cannot chdir to destination directory $dest_dir: $!";
print "Checking jars in directory: " . `pwd`;
chomp(my @jars = `find . -type f -name \*.jar |
+ grep -v build/jars |
grep -v tests\.jar |
grep -v tests-shaded\.jar |
grep -v original |
http://git-wip-us.apache.org/repos/asf/kudu/blob/245f0a7e/java/gradle/shadow.gradle
----------------------------------------------------------------------
diff --git a/java/gradle/shadow.gradle b/java/gradle/shadow.gradle
index 29e22ec..95a9b04 100644
--- a/java/gradle/shadow.gradle
+++ b/java/gradle/shadow.gradle
@@ -53,10 +53,12 @@ shadowJar {
// location across all modules.
shadowJar {
relocate "com.google.common", "org.apache.kudu.shaded.com.google.common"
+ relocate "com.google.gson", "org.apache.kudu.shaded.com.google.gson"
relocate "com.google.protobuf", "org.apache.kudu.shaded.com.google.protobuf"
relocate "com.google.thirdparty", "org.apache.kudu.shaded.com.google.thirdparty"
relocate "com.sangupta", "org.apache.kudu.shaded.com.sangupta"
relocate "org.jboss.netty", "org.apache.kudu.shaded.org.jboss.netty"
+ relocate "scopt", "org.apache.kudu.shaded.scopt"
}
// ------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/245f0a7e/java/kudu-backup/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-backup/pom.xml b/java/kudu-backup/pom.xml
index 4200b5f..0cac985 100644
--- a/java/kudu-backup/pom.xml
+++ b/java/kudu-backup/pom.xml
@@ -184,6 +184,43 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven-shade-plugin.version}</version>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ <include>com.google.gson:protobuf-gson</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.kudu.shaded.com.google.common</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.gson</pattern>
+ <shadedPattern>org.apache.kudu.shaded.com.google.gson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>org.apache.kudu.shaded.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty</pattern>
+ <shadedPattern>org.apache.kudu.shaded.com.google.thirdparty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.github.scopt</pattern>
+ <shadedPattern>org.apache.kudu.shaded.scopt</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>${scala-maven-plugin.version}</version>
[3/3] kudu git commit: Add support for Java dist-tests on Jenkins
Posted by gr...@apache.org.
Add support for Java dist-tests on Jenkins
Enables running the Java dist-tests on Jenkins
when ENABLE_DIST_TEST=1 and BUILD_GRADLE=1.
It requires BUILD_GRADLE=1 because the Maven
build doesn’t support dist-tests.
Change-Id: I20105d3f12fd8c3f95d9fe30fe1b63138614041a
Reviewed-on: http://gerrit.cloudera.org:8080/10954
Reviewed-by: Grant Henke <gr...@apache.org>
Tested-by: Grant Henke <gr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/be360bf0
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/be360bf0
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/be360bf0
Branch: refs/heads/master
Commit: be360bf0122e23f55a12c1ef367cad5329efe274
Parents: 2e1d95c
Author: Grant Henke <gr...@apache.org>
Authored: Fri Aug 24 10:39:10 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Sat Aug 25 03:02:05 2018 +0000
----------------------------------------------------------------------
build-support/dist_test.py | 24 ++++++----
build-support/jenkins/build-and-test.sh | 70 +++++++++++++++++++++-------
build-support/run_dist_test.py | 2 +
3 files changed, 72 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/be360bf0/build-support/dist_test.py
----------------------------------------------------------------------
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index def5dff..ad2bcf1 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -58,6 +58,9 @@ MAX_TASKS_PER_JOB=10000
# of retries, so we have to subtract 1.
FLAKY_TEST_RETRIES = int(os.environ.get('KUDU_FLAKY_TEST_ATTEMPTS', 1)) - 1
+# Flags to include when running Gradle tasks
+GRADLE_FLAGS = os.environ.get('EXTRA_GRADLE_FLAGS', "")
+
PATH_TO_REPO = "../"
# Matches the command line listings in 'ctest -V -N'. For example:
@@ -365,7 +368,8 @@ def create_archive_input(staging, execution,
def create_task_json(staging,
replicate_tasks=1,
- flaky_test_set=set()):
+ flaky_test_set=set(),
+ retry_all_tests=False):
"""
Create a task JSON file suitable for submitting to the distributed
test execution service.
@@ -385,7 +389,7 @@ def create_task_json(staging,
# to get the test name
test_name = ".".join(k.split(".")[:-1])
max_retries = 0
- if test_name in flaky_test_set:
+ if test_name in flaky_test_set or retry_all_tests:
max_retries = FLAKY_TEST_RETRIES
tasks += [{"isolate_hash": str(v),
@@ -528,11 +532,15 @@ def add_loop_test_subparser(subparsers):
def run_java_tests(parser, options):
- subprocess.check_call([rel_to_abs("java/gradlew"), "distTest"],
- cwd=rel_to_abs("java"))
+ subprocess.check_call([rel_to_abs("java/gradlew")] + GRADLE_FLAGS.split() +
+ ["distTest"],
+ cwd=rel_to_abs("java"))
staging = StagingDir(rel_to_abs("java/build/dist-test"))
run_isolate(staging)
- create_task_json(staging, 1)
+ # TODO(ghenke): Add Java tests to the flaky dashboard
+ # KUDU_FLAKY_TEST_LIST doesn't included Java tests.
+ # Instead we will retry all Java tests in case they are flaky.
+ create_task_json(staging, 1, retry_all_tests=True)
submit_tasks(staging, options)
def loop_java_test(parser, options):
@@ -541,9 +549,9 @@ def loop_java_test(parser, options):
"""
if options.num_instances < 1:
parser.error("--num-instances must be >= 1")
- subprocess.check_call(
- [rel_to_abs("java/gradlew"), "distTest", "--classes", "**/%s" % options.pattern],
- cwd=rel_to_abs("java"))
+ subprocess.check_call([rel_to_abs("java/gradlew")] + GRADLE_FLAGS.split() +
+ ["distTest", "--classes", "**/%s" % options.pattern],
+ cwd=rel_to_abs("java"))
staging = StagingDir(rel_to_abs("java/build/dist-test"))
run_isolate(staging)
create_task_json(staging, options.num_instances)
http://git-wip-us.apache.org/repos/asf/kudu/blob/be360bf0/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index 756a2af..352fd17 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -48,10 +48,11 @@
# reporting to work.
#
# ENABLE_DIST_TEST Default: 0
-# If set to 1, will submit C++ tests to be run by the distributed
-# test runner instead of running them locally. This requires that
-# $DIST_TEST_HOME be set to a working dist_test checkout (and that
-# dist_test itself be appropriately configured to point to a cluster)
+# If set to 1, will submit C++ and Java tests to be run by the
+# distributed test runner instead of running them locally.
+# This requires that $DIST_TEST_HOME be set to a working dist_test
+# checkout (and that dist_test itself be appropriately configured to
+# point to a cluster)
#
# BUILD_JAVA Default: 1
# Build and test java code if this is set to 1.
@@ -331,17 +332,18 @@ fi
EXIT_STATUS=0
FAILURES=""
-# If we're running distributed tests, submit them asynchronously while
+# If we're running distributed C++ tests, submit them asynchronously while
# we run the Java and Python tests.
if [ "$ENABLE_DIST_TEST" == "1" ]; then
echo
- echo Submitting distributed-test job.
+ echo Submitting C++ distributed-test job.
echo ------------------------------------------------------------
- export DIST_TEST_JOB_PATH=$BUILD_ROOT/dist-test-job-id
+ # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
+ export DIST_TEST_JOB_PATH=$BUILD_ROOT/c-dist-test-job-id
rm -f $DIST_TEST_JOB_PATH
if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait run ; then
EXIT_STATUS=1
- FAILURES="$FAILURES"$'Could not submit distributed test job\n'
+ FAILURES="$FAILURES"$'Could not submit C++ distributed test job\n'
fi
# Still need to run a few non-dist-test-capable tests locally.
EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -L no_dist_test"
@@ -401,10 +403,24 @@ if [ "$BUILD_JAVA" == "1" ]; then
# incompatibility issue.
EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS -DskipFormat"
EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS $GRADLE_FLAGS"
- # TODO: Run `gradle check` in BUILD_TYPE DEBUG when static code analysis is fixed
- if ! ./gradlew $EXTRA_GRADLE_FLAGS clean test ; then
- EXIT_STATUS=1
- FAILURES="$FAILURES"$'Java Gradle build/test failed\n'
+ # If we're running distributed Java tests, submit them asynchronously.
+ if [ "$ENABLE_DIST_TEST" == "1" ]; then
+ echo
+ echo Submitting Java distributed-test job.
+ echo ------------------------------------------------------------
+ # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
+ export DIST_TEST_JOB_PATH=$BUILD_ROOT/java-dist-test-job-id
+ rm -f $DIST_TEST_JOB_PATH
+ if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait java run-all ; then
+ EXIT_STATUS=1
+ FAILURES="$FAILURES"$'Could not submit Java distributed test job\n'
+ fi
+ else
+ # TODO: Run `gradle check` in BUILD_TYPE DEBUG when static code analysis is fixed
+ if ! ./gradlew $EXTRA_GRADLE_FLAGS clean test ; then
+ EXIT_STATUS=1
+ FAILURES="$FAILURES"$'Java Gradle build/test failed\n'
+ fi
fi
fi
@@ -554,15 +570,16 @@ fi
# If we submitted the tasks earlier, go fetch the results now
if [ "$ENABLE_DIST_TEST" == "1" ]; then
echo
- echo Fetching previously submitted dist-test results...
+ echo Fetching previously submitted C++ dist-test results...
echo ------------------------------------------------------------
- if ! $DIST_TEST_HOME/bin/client watch ; then
+ C_DIST_TEST_ID=`cat $BUILD_ROOT/c-dist-test-job-id`
+ if ! $DIST_TEST_HOME/bin/client watch $C_DIST_TEST_ID ; then
EXIT_STATUS=1
- FAILURES="$FAILURES"$'Distributed tests failed\n'
+ FAILURES="$FAILURES"$'Distributed C++ tests failed\n'
fi
DT_DIR=$TEST_LOGDIR/dist-test-out
rm -Rf $DT_DIR
- $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR
+ $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $C_DIST_TEST_ID
# Fetching the artifacts expands each log into its own directory.
# Move them back into the main log directory
rm -f $DT_DIR/*zip
@@ -570,6 +587,27 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then
mv $arch_dir/build/$BUILD_TYPE_LOWER/test-logs/* $TEST_LOGDIR
rm -Rf $arch_dir
done
+
+ if [ "$BUILD_GRADLE" == "1" ]; then
+ echo
+ echo Fetching previously submitted Java dist-test results...
+ echo ------------------------------------------------------------
+ JAVA_DIST_TEST_ID=`cat $BUILD_ROOT/java-dist-test-job-id`
+ if ! $DIST_TEST_HOME/bin/client watch $JAVA_DIST_TEST_ID ; then
+ EXIT_STATUS=1
+ FAILURES="$FAILURES"$'Distributed Java tests failed\n'
+ fi
+ DT_DIR=$TEST_LOGDIR/java-dist-test-out
+ rm -Rf $DT_DIR
+ $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $JAVA_DIST_TEST_ID
+ # Fetching the artifacts expands each log into its own directory.
+ # Move them back into the main log directory
+ rm -f $DT_DIR/*zip
+ for arch_dir in $DT_DIR/* ; do
+ mv $arch_dir/build/java/test-logs/* $TEST_LOGDIR
+ rm -Rf $arch_dir
+ done
+ fi
fi
if [ $EXIT_STATUS != 0 ]; then
http://git-wip-us.apache.org/repos/asf/kudu/blob/be360bf0/build-support/run_dist_test.py
----------------------------------------------------------------------
diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py
index 2105ff6..297075d 100755
--- a/build-support/run_dist_test.py
+++ b/build-support/run_dist_test.py
@@ -170,6 +170,8 @@ def main():
test_logdir = os.path.abspath(os.path.join(ROOT, "build/java/test-logs"))
if not os.path.exists(test_logdir):
os.makedirs(test_logdir)
+ if not os.path.exists(test_tmpdir):
+ os.makedirs(test_tmpdir)
cmd = [find_java()] + args
stdout = stderr = file(os.path.join(test_logdir, "test-output.txt"), "w")
else:
[2/3] kudu git commit: TestSecurity: remove errant Precondition
Posted by gr...@apache.org.
TestSecurity: remove errant Precondition
Its presence makes it impossible to retry the tests in this class.
An alternative would be to null out miniCluster in tearDown(), but I don't
think that should be necessary for creating a new cluster.
Change-Id: I0ba826d3ec722cb20b35d4c34817d30ab9782976
Reviewed-on: http://gerrit.cloudera.org:8080/11325
Reviewed-by: Dan Burkert <da...@apache.org>
Tested-by: Kudu Jenkins
Reviewed-by: Grant Henke <gr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2e1d95cb
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2e1d95cb
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2e1d95cb
Branch: refs/heads/master
Commit: 2e1d95cb23e18541d610444483717810ec671bb6
Parents: 245f0a7
Author: Adar Dembo <ad...@cloudera.com>
Authored: Fri Aug 24 17:06:13 2018 -0700
Committer: Grant Henke <gr...@apache.org>
Committed: Sat Aug 25 02:33:54 2018 +0000
----------------------------------------------------------------------
.../src/test/java/org/apache/kudu/client/TestSecurity.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/2e1d95cb/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
index 40e4d7d..0ebcc50 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
@@ -65,7 +65,6 @@ public class TestSecurity {
};
private void startCluster(Set<Option> opts) throws IOException {
- Preconditions.checkState(miniCluster == null);
MiniKuduClusterBuilder mcb = new MiniKuduClusterBuilder();
mcb.enableKerberos();
if (opts.contains(Option.LONG_LEADER_ELECTION)) {
@@ -444,4 +443,4 @@ public class TestSecurity {
Assert.assertThat(cla.getAppendedText(), CoreMatchers.containsString(
"Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM."));
}
-}
\ No newline at end of file
+}