You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/05/16 17:41:19 UTC
[1/2] kudu git commit: [java] Remove private method usage in
kudu-mapreduce
Repository: kudu
Updated Branches:
refs/heads/master 0442d47d5 -> da9f97e7f
[java] Remove private method usage in kudu-mapreduce
kudu-mapreduce uses private APIs from KuduPredicate.
This is an issue because the API returns a protobuf
generated class and uses shaded APIs.
This isn’t strictly an issue in the gradle build itself because
calling the shaded methods is ok, but it is an issue in IDEs
like Intellij.
Regardless a shaded dependency is not part of
the public API and should not be exposed/used
because it can cause problems like this.
A follow on patch will work on enforcing that
protobuf is not exposed in the client API so this doesn’t
happen again.
Change-Id: I7ce54f715f970d1c7f9433e39af27d618e73fe6a
Reviewed-on: http://gerrit.cloudera.org:8080/10424
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a587848f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a587848f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a587848f
Branch: refs/heads/master
Commit: a587848f2dae241602da66abfaadb85e4c39a317
Parents: 0442d47
Author: Grant Henke <gr...@apache.org>
Authored: Wed May 16 10:28:19 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Wed May 16 17:40:59 2018 +0000
----------------------------------------------------------------------
.../org/apache/kudu/client/KuduPredicate.java | 34 ++++++++++++++++++++
.../kudu/mapreduce/KuduTableInputFormat.java | 9 ++----
.../kudu/mapreduce/KuduTableMapReduceUtil.java | 7 ++--
3 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/a587848f/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
index 997c50b..b0ff0b4 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
@@ -17,6 +17,9 @@
package org.apache.kudu.client;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.math.BigInteger;
import java.math.BigDecimal;
import java.util.ArrayList;
@@ -765,6 +768,37 @@ public class KuduPredicate {
}
/**
+ * Serializes a list of {@code KuduPredicate} into a byte array.
+ * @return the serialized kudu predicates
+ * @throws IOException
+ */
+ @InterfaceAudience.LimitedPrivate("kudu-mapreduce")
+ public static byte[] serialize(List<KuduPredicate> predicates) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (KuduPredicate predicate : predicates) {
+ Common.ColumnPredicatePB message = predicate.toPB();
+ message.writeDelimitedTo(baos);
+ }
+ return baos.toByteArray();
+ }
+
+ /**
+ * Serializes a list of {@code KuduPredicate} into a byte array.
+ * @return the serialized kudu predicates
+ * @throws IOException
+ */
+ @InterfaceAudience.LimitedPrivate("kudu-mapreduce")
+ public static List<KuduPredicate> deserialize(Schema schema, byte[] bytes) throws IOException {
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ List<KuduPredicate> predicates = new ArrayList<>();
+ while (bais.available() > 0) {
+ Common.ColumnPredicatePB message = Common.ColumnPredicatePB.parseDelimitedFrom(bais);
+ predicates.add(KuduPredicate.fromPB(schema, message));
+ }
+ return predicates;
+ }
+
+ /**
* Convert the predicate to the protobuf representation.
* @return the protobuf message for this predicate
*/
http://git-wip-us.apache.org/repos/asf/kudu/blob/a587848f/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
index cbf12f6..c09f805 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
@@ -251,14 +251,9 @@ public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult>
}
}
- this.predicates = new ArrayList<>();
try {
- InputStream is =
- new ByteArrayInputStream(Base64.decodeBase64(conf.get(ENCODED_PREDICATES_KEY, "")));
- while (is.available() > 0) {
- this.predicates.add(KuduPredicate.fromPB(table.getSchema(),
- Common.ColumnPredicatePB.parseDelimitedFrom(is)));
- }
+ byte[] bytes = Base64.decodeBase64(conf.get(ENCODED_PREDICATES_KEY, ""));
+ this.predicates = KuduPredicate.deserialize(table.getSchema(), bytes);
} catch (IOException e) {
throw new RuntimeException("unable to deserialize predicates from the configuration", e);
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/a587848f/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
index d2689ce..52b2fac 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
@@ -258,11 +258,8 @@ public class KuduTableMapReduceUtil {
* @return the encoded predicates
*/
static String base64EncodePredicates(List<KuduPredicate> predicates) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- for (KuduPredicate predicate : predicates) {
- predicate.toPB().writeDelimitedTo(baos);
- }
- return Base64.encodeBase64String(baos.toByteArray());
+ byte[] bytes = KuduPredicate.serialize(predicates);
+ return Base64.encodeBase64String(bytes);
}
[2/2] kudu git commit: [java] Add kudu-client dep to kudu-spark-tools
Posted by gr...@apache.org.
[java] Add kudu-client dep to kudu-spark-tools
We use the kudu-client APIs in kudu-spark-tools
but don’t define it as an explicit dependency.
Using transitive dependencies is a bad practice
but generally doesn’t break anything. However,
the Gradle Intellij integration seams to not find
the kudu-client project dependnecy if it’s not
explicitly listed here.
Change-Id: I6255bedac9781981ae8a937f0dc72afff0fc5ef0
Reviewed-on: http://gerrit.cloudera.org:8080/10425
Tested-by: Grant Henke <gr...@apache.org>
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/da9f97e7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/da9f97e7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/da9f97e7
Branch: refs/heads/master
Commit: da9f97e7fe492c7493f0c7e1027cb1e701002353
Parents: a587848
Author: Grant Henke <gr...@apache.org>
Authored: Wed May 16 11:12:09 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Wed May 16 17:41:07 2018 +0000
----------------------------------------------------------------------
java/kudu-spark-tools/build.gradle | 1 +
java/kudu-spark-tools/pom.xml | 5 +++++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/da9f97e7/java/kudu-spark-tools/build.gradle
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/build.gradle b/java/kudu-spark-tools/build.gradle
index 606d8ed..0f2cdd7 100644
--- a/java/kudu-spark-tools/build.gradle
+++ b/java/kudu-spark-tools/build.gradle
@@ -19,6 +19,7 @@ apply plugin: "scala"
apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
+ compile project(path: ":kudu-client", configuration: "shadow")
compile project(path: ":kudu-client-tools", configuration: "shadow")
compile project(path: ":kudu-spark", configuration: "shadow")
http://git-wip-us.apache.org/repos/asf/kudu/blob/da9f97e7/java/kudu-spark-tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/pom.xml b/java/kudu-spark-tools/pom.xml
index 2bc79f0..fbfc2b5 100644
--- a/java/kudu-spark-tools/pom.xml
+++ b/java/kudu-spark-tools/pom.xml
@@ -34,6 +34,11 @@
<dependencies>
<dependency>
<groupId>org.apache.kudu</groupId>
+ <artifactId>kudu-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client-tools</artifactId>
<version>${project.version}</version>
</dependency>