You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2019/06/07 21:41:19 UTC

[kudu] 03/03: [backup] Fix backup CLI dependencies

This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit b4128848ccc8f56299d181c4a193970c2f59429a
Author: Grant Henke <gr...@apache.org>
AuthorDate: Thu Jun 6 17:17:14 2019 -0500

    [backup] Fix backup CLI dependencies
    
    This patch fixes the backup dependencies in two ways.
    
    First, it fixes the Gradle handling of `isTool` to ensure
    sl4j is included.
    
    Second it breaks out the common
    “library” type functionality into a kudu-backup-common
    module so it can be shared between the kudu-back and
    kudu-backup-tools module. This allows
    kudu-backup-tools to shade slf4j and log4j without
    affecting the kudu-backup dependencies.
    
    I tested this by running a build and then running the
    kudu-backup-tools localy via:
       `java -cp build/libs/kudu-backup-tools-1.10.0-SNAPSHOT.jar org.apache.kudu.backup.KuduBackupCLI`
    
    The patch also sneaks in some interface annotations.
    
    Change-Id: I6bee1f467d5652ced8412eef0315057e5b0d684e
    Reviewed-on: http://gerrit.cloudera.org:8080/13547
    Tested-by: Grant Henke <gr...@apache.org>
    Reviewed-by: Mike Percy <mp...@apache.org>
---
 java/gradle/shadow.gradle                          | 35 +++++++++++-----------
 .../build.gradle                                   | 20 ++++---------
 .../src/main/protobuf/backup.proto                 |  0
 .../scala/org/apache/kudu/backup/BackupGraph.scala |  4 +++
 .../scala/org/apache/kudu/backup/BackupIO.scala    |  0
 .../org/apache/kudu/backup/TableMetadata.scala     |  0
 .../src/test/resources/log4j2.properties           | 32 ++++++++++++++++++++
 .../org/apache/kudu/backup/TestBackupGraph.scala   |  0
 java/kudu-backup-tools/build.gradle                | 24 ++++-----------
 .../src/main/resources/log4j2.properties           | 29 ++++++++++++++++++
 .../org/apache/kudu/backup/KuduBackupCLI.scala     | 10 +++++++
 .../org/apache/kudu/backup/KuduBackupCleaner.scala |  5 +++-
 java/kudu-backup/build.gradle                      |  2 +-
 java/settings.gradle                               |  1 +
 14 files changed, 111 insertions(+), 51 deletions(-)

diff --git a/java/gradle/shadow.gradle b/java/gradle/shadow.gradle
index 63e7c11..499554f 100644
--- a/java/gradle/shadow.gradle
+++ b/java/gradle/shadow.gradle
@@ -41,23 +41,6 @@ configurations.archives.artifacts.removeAll {
   it instanceof ArchivePublishArtifact && it.archiveTask == jar
 }
 
-// Define an overridable property to indicate tool jars that should
-// include all of their dependencies.
-// We use this below to ensure slf4j is included.
-shadow.ext {
-  isToolJar = false
-}
-if (!shadow.isToolJar) {
-  shadowJar {
-    dependencies {
-      // Our shaded library jars always try to pull in the slf4j api from
-      // kudu-client, though we never want it included. Excluding it
-      // here prevents the need to always list it.
-      exclude dependency(libs.slf4jApi)
-    }
-  }
-}
-
 // Ensure we always relocate these shaded dependencies to the same
 // location across all modules.
 shadowJar {
@@ -84,9 +67,27 @@ configurations.create("compileUnshaded")
 configurations.shadow.extendsFrom(configurations.compileUnshaded)
 configurations.compile.extendsFrom(configurations.compileUnshaded)
 
+// Define an overridable property to indicate tool jars that should
+// include all of their dependencies.
+// We use this below to ensure slf4j is included.
+shadow.ext {
+  isToolJar = false
+}
+
 // We use afterEvaluate to add additional configuration once all the definitions
 // in the projects build script have been applied
 afterEvaluate {
+  if (!shadow.isToolJar) {
+    shadowJar {
+      dependencies {
+        // Our shaded library jars always try to pull in the slf4j api from
+        // kudu-client, though we never want it included. Excluding it
+        // here prevents the need to always list it.
+        exclude dependency(libs.slf4jApi)
+      }
+    }
+  }
+  
   // Ensure compileUnshaded dependencies are included in the pom.
   [install, uploadArchives].each { task ->
     task.repositories.each {
diff --git a/java/kudu-backup-tools/build.gradle b/java/kudu-backup-common/build.gradle
similarity index 79%
copy from java/kudu-backup-tools/build.gradle
copy to java/kudu-backup-common/build.gradle
index d6ae503..e4d95e9 100644
--- a/java/kudu-backup-tools/build.gradle
+++ b/java/kudu-backup-common/build.gradle
@@ -17,12 +17,6 @@
 
 apply plugin: "scala"
 apply from: "$rootDir/gradle/protobuf.gradle"
-apply from: "$rootDir/gradle/shadow.gradle"
-
-// Mark this as a tool jar so shadow doesn't exclude any dependencies.
-shadow {
-  isToolJar = true
-}
 
 dependencies {
   compile project(path: ":kudu-client", configuration: "shadow")
@@ -32,10 +26,6 @@ dependencies {
     exclude group: "com.google.guava", module: "guava"
   }
   compile libs.hadoopCommon
-  compile (libs.scopt)  {
-    // Make sure wrong Scala version is not pulled in.
-    exclude group: "org.scala-lang", module: "scala-library"
-  }
   compile libs.scalaLibrary
   compile libs.slf4jApi
 
@@ -43,8 +33,6 @@ dependencies {
 
   testCompile project(path: ":kudu-test-utils", configuration: "shadow")
   testCompile libs.junit
-  testCompile libs.log4j
-  testCompile libs.log4jSlf4jImpl
   testCompile libs.scalatest
 }
 
@@ -57,7 +45,11 @@ sourceSets {
   }
 }
 
-// kudu-backup-tools has no public Javadoc.
+// kudu-backup-common has no public Javadoc.
 javadoc {
   enabled = false
-}
\ No newline at end of file
+}
+
+// Skip publishing kudu-backup-common artifact because it will always be shaded into
+// kudu-backup and kudu-backup-tools.
+uploadArchives.enabled = false
\ No newline at end of file
diff --git a/java/kudu-backup-tools/src/main/protobuf/backup.proto b/java/kudu-backup-common/src/main/protobuf/backup.proto
similarity index 100%
rename from java/kudu-backup-tools/src/main/protobuf/backup.proto
rename to java/kudu-backup-common/src/main/protobuf/backup.proto
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupGraph.scala b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
similarity index 98%
rename from java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
rename to java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
index be6207f..6e0ecaf 100644
--- a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
+++ b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
@@ -171,6 +171,8 @@ class BackupGraph(val tableId: String) {
 /**
  * Node class to represent nodes in the backup graph.
  */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 case class BackupNode(path: Path, metadata: TableMetadataPB) {
 
   /**
@@ -185,6 +187,8 @@ case class BackupNode(path: Path, metadata: TableMetadataPB) {
 /**
  * A backup path is a full backup with a series of incremental backups.
  */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 case class BackupPath(backups: Seq[BackupNode]) {
 
   def fullBackup: BackupNode = backups.head
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupIO.scala b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupIO.scala
similarity index 100%
rename from java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupIO.scala
rename to java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupIO.scala
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/TableMetadata.scala b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
similarity index 100%
rename from java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
rename to java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
diff --git a/java/kudu-backup-common/src/test/resources/log4j2.properties b/java/kudu-backup-common/src/test/resources/log4j2.properties
new file mode 100644
index 0000000..e85deec
--- /dev/null
+++ b/java/kudu-backup-common/src/test/resources/log4j2.properties
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+status = error
+name = PropertiesConfig
+appenders = console
+
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+
+rootLogger.level = info
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+logger.kudu.name = org.apache.kudu
+logger.kudu.level = debug
\ No newline at end of file
diff --git a/java/kudu-backup-tools/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala b/java/kudu-backup-common/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala
similarity index 100%
rename from java/kudu-backup-tools/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala
rename to java/kudu-backup-common/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala
diff --git a/java/kudu-backup-tools/build.gradle b/java/kudu-backup-tools/build.gradle
index d6ae503..df89c76 100644
--- a/java/kudu-backup-tools/build.gradle
+++ b/java/kudu-backup-tools/build.gradle
@@ -16,47 +16,35 @@
 // under the License.
 
 apply plugin: "scala"
-apply from: "$rootDir/gradle/protobuf.gradle"
 apply from: "$rootDir/gradle/shadow.gradle"
 
-// Mark this as a tool jar so shadow doesn't exclude any dependencies.
+// Mark this as a tool jar so shadow doesn't exclude any dependencies when shading.
 shadow {
   isToolJar = true
 }
 
 dependencies {
+  compile project(path: ":kudu-backup-common")
   compile project(path: ":kudu-client", configuration: "shadow")
-  compile libs.protobufJava
-  compile (libs.protobufJavaUtil) {
-    // Make sure wrong Guava version is not pulled in.
-    exclude group: "com.google.guava", module: "guava"
-  }
   compile libs.hadoopCommon
   compile (libs.scopt)  {
     // Make sure wrong Scala version is not pulled in.
     exclude group: "org.scala-lang", module: "scala-library"
   }
   compile libs.scalaLibrary
+
+  // Include the logging implementation given this is a standalone command line tool.
   compile libs.slf4jApi
+  compile libs.log4j
+  compile libs.log4jSlf4jImpl
 
   optional libs.yetusAnnotations
 
   testCompile project(path: ":kudu-test-utils", configuration: "shadow")
   testCompile libs.junit
-  testCompile libs.log4j
-  testCompile libs.log4jSlf4jImpl
   testCompile libs.scalatest
 }
 
-// Add protobuf files to the proto source set.
-sourceSets {
-  main {
-    proto {
-      srcDir "src/main/protobuf"
-    }
-  }
-}
-
 // kudu-backup-tools has no public Javadoc.
 javadoc {
   enabled = false
diff --git a/java/kudu-backup-tools/src/main/resources/log4j2.properties b/java/kudu-backup-tools/src/main/resources/log4j2.properties
new file mode 100644
index 0000000..881d729
--- /dev/null
+++ b/java/kudu-backup-tools/src/main/resources/log4j2.properties
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+status = error
+name = PropertiesConfig
+appenders = console
+
+appender.console.type = Console
+appender.console.name = STDERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+
+rootLogger.level = error
+rootLogger.appenderRefs = stderr
+rootLogger.appenderRef.stderr.ref = STDERR
\ No newline at end of file
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala
index c77abc3..6c2123b 100644
--- a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala
+++ b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala
@@ -19,18 +19,26 @@ package org.apache.kudu.backup
 import java.text.SimpleDateFormat
 
 import org.apache.hadoop.conf.Configuration
+import org.apache.yetus.audience.InterfaceAudience
+import org.apache.yetus.audience.InterfaceStability
 import scopt.OptionParser
 
 // The possible backup CLI tool actions.
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 object Action extends Enumeration {
   val LIST_LATEST, LIST_RESTORE_SEQUENCE, LIST_ALL = Value
 }
 
 // The possible backup CLI print formats.
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 object Format extends Enumeration {
   val PRETTY, TSV, CSV = Value
 }
 
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 case class BackupCLIOptions(
     action: Action.Value,
     format: Format.Value,
@@ -85,6 +93,8 @@ object BackupCLIOptions {
   }
 }
 
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 object KuduBackupCLI {
 
   // The header for all tables printed by the tool.
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala
index 8462aa9..0170c6e 100644
--- a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala
+++ b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala
@@ -22,9 +22,12 @@ import java.time.Instant
 
 import org.apache.hadoop.conf.Configuration
 import scopt.OptionParser
-
 import org.apache.kudu.backup.Backup.TableMetadataPB
+import org.apache.yetus.audience.InterfaceAudience
+import org.apache.yetus.audience.InterfaceStability
 
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 case class BackupCleanerOptions(
     tables: Seq[String],
     rootPath: String,
diff --git a/java/kudu-backup/build.gradle b/java/kudu-backup/build.gradle
index 8988aa7..47d414e 100644
--- a/java/kudu-backup/build.gradle
+++ b/java/kudu-backup/build.gradle
@@ -20,7 +20,7 @@ apply from: "$rootDir/gradle/shadow.gradle"
 
 dependencies {
   // Note: We don't use the shaded version, so we can control the dependencies.
-  compile(project(path: ":kudu-backup-tools")) {
+  compile(project(path: ":kudu-backup-common")) {
     // Ensure we use the hadoop-client provided by Spark to avoid any compatibility issues.
     exclude group: "org.apache.hadoop", module: "hadoop-common"
   }
diff --git a/java/settings.gradle b/java/settings.gradle
index 145c3c8..8a8bcfc 100644
--- a/java/settings.gradle
+++ b/java/settings.gradle
@@ -20,6 +20,7 @@
 
 rootProject.name = "kudu-parent"
 include "kudu-backup"
+include "kudu-backup-common"
 include "kudu-backup-tools"
 include "kudu-client"
 include "kudu-client-tools"