You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2019/06/07 21:41:19 UTC
[kudu] 03/03: [backup] Fix backup CLI dependencies
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit b4128848ccc8f56299d181c4a193970c2f59429a
Author: Grant Henke <gr...@apache.org>
AuthorDate: Thu Jun 6 17:17:14 2019 -0500
[backup] Fix backup CLI dependencies
This patch fixes the backup dependencies in two ways.
First, it fixes the Gradle handling of `isTool` to ensure
sl4j is included.
Second it breaks out the common
“library” type functionality into a kudu-backup-common
module so it can be shared between the kudu-back and
kudu-backup-tools module. This allows
kudu-backup-tools to shade slf4j and log4j without
affecting the kudu-backup dependencies.
I tested this by running a build and then running the
kudu-backup-tools localy via:
`java -cp build/libs/kudu-backup-tools-1.10.0-SNAPSHOT.jar org.apache.kudu.backup.KuduBackupCLI`
The patch also sneaks in some interface annotations.
Change-Id: I6bee1f467d5652ced8412eef0315057e5b0d684e
Reviewed-on: http://gerrit.cloudera.org:8080/13547
Tested-by: Grant Henke <gr...@apache.org>
Reviewed-by: Mike Percy <mp...@apache.org>
---
java/gradle/shadow.gradle | 35 +++++++++++-----------
.../build.gradle | 20 ++++---------
.../src/main/protobuf/backup.proto | 0
.../scala/org/apache/kudu/backup/BackupGraph.scala | 4 +++
.../scala/org/apache/kudu/backup/BackupIO.scala | 0
.../org/apache/kudu/backup/TableMetadata.scala | 0
.../src/test/resources/log4j2.properties | 32 ++++++++++++++++++++
.../org/apache/kudu/backup/TestBackupGraph.scala | 0
java/kudu-backup-tools/build.gradle | 24 ++++-----------
.../src/main/resources/log4j2.properties | 29 ++++++++++++++++++
.../org/apache/kudu/backup/KuduBackupCLI.scala | 10 +++++++
.../org/apache/kudu/backup/KuduBackupCleaner.scala | 5 +++-
java/kudu-backup/build.gradle | 2 +-
java/settings.gradle | 1 +
14 files changed, 111 insertions(+), 51 deletions(-)
diff --git a/java/gradle/shadow.gradle b/java/gradle/shadow.gradle
index 63e7c11..499554f 100644
--- a/java/gradle/shadow.gradle
+++ b/java/gradle/shadow.gradle
@@ -41,23 +41,6 @@ configurations.archives.artifacts.removeAll {
it instanceof ArchivePublishArtifact && it.archiveTask == jar
}
-// Define an overridable property to indicate tool jars that should
-// include all of their dependencies.
-// We use this below to ensure slf4j is included.
-shadow.ext {
- isToolJar = false
-}
-if (!shadow.isToolJar) {
- shadowJar {
- dependencies {
- // Our shaded library jars always try to pull in the slf4j api from
- // kudu-client, though we never want it included. Excluding it
- // here prevents the need to always list it.
- exclude dependency(libs.slf4jApi)
- }
- }
-}
-
// Ensure we always relocate these shaded dependencies to the same
// location across all modules.
shadowJar {
@@ -84,9 +67,27 @@ configurations.create("compileUnshaded")
configurations.shadow.extendsFrom(configurations.compileUnshaded)
configurations.compile.extendsFrom(configurations.compileUnshaded)
+// Define an overridable property to indicate tool jars that should
+// include all of their dependencies.
+// We use this below to ensure slf4j is included.
+shadow.ext {
+ isToolJar = false
+}
+
// We use afterEvaluate to add additional configuration once all the definitions
// in the projects build script have been applied
afterEvaluate {
+ if (!shadow.isToolJar) {
+ shadowJar {
+ dependencies {
+ // Our shaded library jars always try to pull in the slf4j api from
+ // kudu-client, though we never want it included. Excluding it
+ // here prevents the need to always list it.
+ exclude dependency(libs.slf4jApi)
+ }
+ }
+ }
+
// Ensure compileUnshaded dependencies are included in the pom.
[install, uploadArchives].each { task ->
task.repositories.each {
diff --git a/java/kudu-backup-tools/build.gradle b/java/kudu-backup-common/build.gradle
similarity index 79%
copy from java/kudu-backup-tools/build.gradle
copy to java/kudu-backup-common/build.gradle
index d6ae503..e4d95e9 100644
--- a/java/kudu-backup-tools/build.gradle
+++ b/java/kudu-backup-common/build.gradle
@@ -17,12 +17,6 @@
apply plugin: "scala"
apply from: "$rootDir/gradle/protobuf.gradle"
-apply from: "$rootDir/gradle/shadow.gradle"
-
-// Mark this as a tool jar so shadow doesn't exclude any dependencies.
-shadow {
- isToolJar = true
-}
dependencies {
compile project(path: ":kudu-client", configuration: "shadow")
@@ -32,10 +26,6 @@ dependencies {
exclude group: "com.google.guava", module: "guava"
}
compile libs.hadoopCommon
- compile (libs.scopt) {
- // Make sure wrong Scala version is not pulled in.
- exclude group: "org.scala-lang", module: "scala-library"
- }
compile libs.scalaLibrary
compile libs.slf4jApi
@@ -43,8 +33,6 @@ dependencies {
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile libs.junit
- testCompile libs.log4j
- testCompile libs.log4jSlf4jImpl
testCompile libs.scalatest
}
@@ -57,7 +45,11 @@ sourceSets {
}
}
-// kudu-backup-tools has no public Javadoc.
+// kudu-backup-common has no public Javadoc.
javadoc {
enabled = false
-}
\ No newline at end of file
+}
+
+// Skip publishing kudu-backup-common artifact because it will always be shaded into
+// kudu-backup and kudu-backup-tools.
+uploadArchives.enabled = false
\ No newline at end of file
diff --git a/java/kudu-backup-tools/src/main/protobuf/backup.proto b/java/kudu-backup-common/src/main/protobuf/backup.proto
similarity index 100%
rename from java/kudu-backup-tools/src/main/protobuf/backup.proto
rename to java/kudu-backup-common/src/main/protobuf/backup.proto
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupGraph.scala b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
similarity index 98%
rename from java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
rename to java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
index be6207f..6e0ecaf 100644
--- a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
+++ b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
@@ -171,6 +171,8 @@ class BackupGraph(val tableId: String) {
/**
* Node class to represent nodes in the backup graph.
*/
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
case class BackupNode(path: Path, metadata: TableMetadataPB) {
/**
@@ -185,6 +187,8 @@ case class BackupNode(path: Path, metadata: TableMetadataPB) {
/**
* A backup path is a full backup with a series of incremental backups.
*/
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
case class BackupPath(backups: Seq[BackupNode]) {
def fullBackup: BackupNode = backups.head
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupIO.scala b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupIO.scala
similarity index 100%
rename from java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/BackupIO.scala
rename to java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/BackupIO.scala
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/TableMetadata.scala b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
similarity index 100%
rename from java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
rename to java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
diff --git a/java/kudu-backup-common/src/test/resources/log4j2.properties b/java/kudu-backup-common/src/test/resources/log4j2.properties
new file mode 100644
index 0000000..e85deec
--- /dev/null
+++ b/java/kudu-backup-common/src/test/resources/log4j2.properties
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+status = error
+name = PropertiesConfig
+appenders = console
+
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+
+rootLogger.level = info
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+logger.kudu.name = org.apache.kudu
+logger.kudu.level = debug
\ No newline at end of file
diff --git a/java/kudu-backup-tools/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala b/java/kudu-backup-common/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala
similarity index 100%
rename from java/kudu-backup-tools/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala
rename to java/kudu-backup-common/src/test/scala/org/apache/kudu/backup/TestBackupGraph.scala
diff --git a/java/kudu-backup-tools/build.gradle b/java/kudu-backup-tools/build.gradle
index d6ae503..df89c76 100644
--- a/java/kudu-backup-tools/build.gradle
+++ b/java/kudu-backup-tools/build.gradle
@@ -16,47 +16,35 @@
// under the License.
apply plugin: "scala"
-apply from: "$rootDir/gradle/protobuf.gradle"
apply from: "$rootDir/gradle/shadow.gradle"
-// Mark this as a tool jar so shadow doesn't exclude any dependencies.
+// Mark this as a tool jar so shadow doesn't exclude any dependencies when shading.
shadow {
isToolJar = true
}
dependencies {
+ compile project(path: ":kudu-backup-common")
compile project(path: ":kudu-client", configuration: "shadow")
- compile libs.protobufJava
- compile (libs.protobufJavaUtil) {
- // Make sure wrong Guava version is not pulled in.
- exclude group: "com.google.guava", module: "guava"
- }
compile libs.hadoopCommon
compile (libs.scopt) {
// Make sure wrong Scala version is not pulled in.
exclude group: "org.scala-lang", module: "scala-library"
}
compile libs.scalaLibrary
+
+ // Include the logging implementation given this is a standalone command line tool.
compile libs.slf4jApi
+ compile libs.log4j
+ compile libs.log4jSlf4jImpl
optional libs.yetusAnnotations
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile libs.junit
- testCompile libs.log4j
- testCompile libs.log4jSlf4jImpl
testCompile libs.scalatest
}
-// Add protobuf files to the proto source set.
-sourceSets {
- main {
- proto {
- srcDir "src/main/protobuf"
- }
- }
-}
-
// kudu-backup-tools has no public Javadoc.
javadoc {
enabled = false
diff --git a/java/kudu-backup-tools/src/main/resources/log4j2.properties b/java/kudu-backup-tools/src/main/resources/log4j2.properties
new file mode 100644
index 0000000..881d729
--- /dev/null
+++ b/java/kudu-backup-tools/src/main/resources/log4j2.properties
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+status = error
+name = PropertiesConfig
+appenders = console
+
+appender.console.type = Console
+appender.console.name = STDERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+
+rootLogger.level = error
+rootLogger.appenderRefs = stderr
+rootLogger.appenderRef.stderr.ref = STDERR
\ No newline at end of file
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala
index c77abc3..6c2123b 100644
--- a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala
+++ b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCLI.scala
@@ -19,18 +19,26 @@ package org.apache.kudu.backup
import java.text.SimpleDateFormat
import org.apache.hadoop.conf.Configuration
+import org.apache.yetus.audience.InterfaceAudience
+import org.apache.yetus.audience.InterfaceStability
import scopt.OptionParser
// The possible backup CLI tool actions.
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
object Action extends Enumeration {
val LIST_LATEST, LIST_RESTORE_SEQUENCE, LIST_ALL = Value
}
// The possible backup CLI print formats.
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
object Format extends Enumeration {
val PRETTY, TSV, CSV = Value
}
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
case class BackupCLIOptions(
action: Action.Value,
format: Format.Value,
@@ -85,6 +93,8 @@ object BackupCLIOptions {
}
}
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
object KuduBackupCLI {
// The header for all tables printed by the tool.
diff --git a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala
index 8462aa9..0170c6e 100644
--- a/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala
+++ b/java/kudu-backup-tools/src/main/scala/org/apache/kudu/backup/KuduBackupCleaner.scala
@@ -22,9 +22,12 @@ import java.time.Instant
import org.apache.hadoop.conf.Configuration
import scopt.OptionParser
-
import org.apache.kudu.backup.Backup.TableMetadataPB
+import org.apache.yetus.audience.InterfaceAudience
+import org.apache.yetus.audience.InterfaceStability
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
case class BackupCleanerOptions(
tables: Seq[String],
rootPath: String,
diff --git a/java/kudu-backup/build.gradle b/java/kudu-backup/build.gradle
index 8988aa7..47d414e 100644
--- a/java/kudu-backup/build.gradle
+++ b/java/kudu-backup/build.gradle
@@ -20,7 +20,7 @@ apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
// Note: We don't use the shaded version, so we can control the dependencies.
- compile(project(path: ":kudu-backup-tools")) {
+ compile(project(path: ":kudu-backup-common")) {
// Ensure we use the hadoop-client provided by Spark to avoid any compatibility issues.
exclude group: "org.apache.hadoop", module: "hadoop-common"
}
diff --git a/java/settings.gradle b/java/settings.gradle
index 145c3c8..8a8bcfc 100644
--- a/java/settings.gradle
+++ b/java/settings.gradle
@@ -20,6 +20,7 @@
rootProject.name = "kudu-parent"
include "kudu-backup"
+include "kudu-backup-common"
include "kudu-backup-tools"
include "kudu-client"
include "kudu-client-tools"