You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2018/10/19 19:24:10 UTC
[1/2] kudu git commit: Add ExtractDouble method to JsonReader and
small improvements
Repository: kudu
Updated Branches:
refs/heads/master 785490ce5 -> b3486d99a
Add ExtractDouble method to JsonReader and small improvements
Change-Id: Iccbdc2467746b68ac871f5feeb01e610610b13d6
Reviewed-on: http://gerrit.cloudera.org:8080/11728
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0a7940d6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0a7940d6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0a7940d6
Branch: refs/heads/master
Commit: 0a7940d687980d1451cd70f7a4b5b80e62797808
Parents: 785490c
Author: Will Berkeley <wd...@gmail.org>
Authored: Thu Oct 18 14:24:35 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Fri Oct 19 18:34:46 2018 +0000
----------------------------------------------------------------------
src/kudu/tools/kudu-admin-test.cc | 2 +-
src/kudu/util/jsonreader-test.cc | 42 ++++++++++++++++++++
src/kudu/util/jsonreader.cc | 72 ++++++++++++++++++++++++++--------
src/kudu/util/jsonreader.h | 4 ++
4 files changed, 103 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/0a7940d6/src/kudu/tools/kudu-admin-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc
index 22612ac..78adeeb 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -1976,7 +1976,7 @@ TEST_F(AdminCliTest, TestLocateRowMore) {
// Test providing valid JSON that's not an array.
NO_FATALS(check_bad_input(
"{ \"key_hash\" : \"foo\", \"key_range\" : 2 }",
- "Wrong type during field extraction: expected object array"));
+ "wrong type during field extraction: expected object array"));
}
TEST_F(AdminCliTest, TestLocateRowAndCheckRowPresence) {
http://git-wip-us.apache.org/repos/asf/kudu/blob/0a7940d6/src/kudu/util/jsonreader-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/jsonreader-test.cc b/src/kudu/util/jsonreader-test.cc
index 50e1fc6..250091f 100644
--- a/src/kudu/util/jsonreader-test.cc
+++ b/src/kudu/util/jsonreader-test.cc
@@ -56,6 +56,9 @@ TEST(JsonReaderTest, Empty) {
ASSERT_TRUE(r.ExtractBool(r.root(), "foo", nullptr).IsNotFound());
ASSERT_TRUE(r.ExtractInt32(r.root(), "foo", nullptr).IsNotFound());
ASSERT_TRUE(r.ExtractInt64(r.root(), "foo", nullptr).IsNotFound());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), "foo", nullptr).IsNotFound());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), "foo", nullptr).IsNotFound());
+ ASSERT_TRUE(r.ExtractDouble(r.root(), "foo", nullptr).IsNotFound());
ASSERT_TRUE(r.ExtractString(r.root(), "foo", nullptr).IsNotFound());
ASSERT_TRUE(r.ExtractObject(r.root(), "foo", nullptr).IsNotFound());
ASSERT_TRUE(r.ExtractObjectArray(r.root(), "foo", nullptr).IsNotFound());
@@ -72,6 +75,9 @@ TEST(JsonReaderTest, Basic) {
ASSERT_TRUE(r.ExtractBool(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt32(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt64(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractDouble(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObject(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObjectArray(r.root(), "foo", nullptr).IsInvalidArgument());
}
@@ -112,17 +118,26 @@ TEST(JsonReaderTest, LessBasic) {
ASSERT_TRUE(r.ExtractBool(r.root(), "null", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt32(r.root(), "null", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt64(r.root(), "null", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), "null", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), "null", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractDouble(r.root(), "null", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObject(r.root(), "null", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObjectArray(r.root(), "null", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractBool(r.root(), "empty", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt32(r.root(), "empty", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt64(r.root(), "empty", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), "empty", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), "empty", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractDouble(r.root(), "empty", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObject(r.root(), "empty", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObjectArray(r.root(), "empty", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt32(r.root(), "bool", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt64(r.root(), "bool", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), "bool", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), "bool", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractDouble(r.root(), "bool", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractString(r.root(), "bool", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObject(r.root(), "bool", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObjectArray(r.root(), "bool", nullptr).IsInvalidArgument());
@@ -186,6 +201,7 @@ TEST(JsonReaderTest, SignedAndUnsignedInts) {
uint64_t signed_big64_uint64;
ASSERT_OK(r.ExtractUint64(r.root(), signed_big64, &signed_big64_uint64));
ASSERT_EQ(kMaxInt64, signed_big64_uint64);
+ ASSERT_TRUE(r.ExtractDouble(r.root(), signed_big64, nullptr).IsInvalidArgument());
// Max unsigned 32-bit integer.
const char* const unsigned_big32 = "unsigned_big32";
@@ -208,6 +224,7 @@ TEST(JsonReaderTest, SignedAndUnsignedInts) {
uint64_t unsigned_big64_uint64;
ASSERT_OK(r.ExtractUint64(r.root(), unsigned_big64, &unsigned_big64_uint64));
ASSERT_EQ(kMaxUint64, unsigned_big64_uint64);
+ ASSERT_TRUE(r.ExtractDouble(r.root(), unsigned_big64, nullptr).IsInvalidArgument());
// Min signed 32-bit integer.
const char* const signed_small32 = "signed_small32";
@@ -230,6 +247,25 @@ TEST(JsonReaderTest, SignedAndUnsignedInts) {
ASSERT_TRUE(r.ExtractUint64(r.root(), signed_small64, nullptr).IsInvalidArgument());
}
+TEST(JsonReaderTest, Doubles) {
+ JsonReader r("{ \"foo\" : 5.125 }");
+ ASSERT_OK(r.Init());
+
+ double foo;
+ ASSERT_OK(r.ExtractDouble(r.root(), "foo", &foo));
+ ASSERT_EQ(5.125, foo);
+
+ // Bad types.
+ ASSERT_TRUE(r.ExtractBool(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractInt32(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractInt64(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractString(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractObject(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractObjectArray(r.root(), "foo", nullptr).IsInvalidArgument());
+}
+
TEST(JsonReaderTest, Objects) {
JsonReader r("{ \"foo\" : { \"1\" : 1 } }");
ASSERT_OK(r.Init());
@@ -246,6 +282,9 @@ TEST(JsonReaderTest, Objects) {
ASSERT_TRUE(r.ExtractBool(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt32(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt64(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), "foo", nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractDouble(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractString(r.root(), "foo", nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObjectArray(r.root(), "foo", nullptr).IsInvalidArgument());
}
@@ -267,6 +306,9 @@ TEST(JsonReaderTest, TopLevelArray) {
ASSERT_TRUE(r.ExtractBool(r.root(), nullptr, nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt32(r.root(), nullptr, nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractInt64(r.root(), nullptr, nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint32(r.root(), nullptr, nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractUint64(r.root(), nullptr, nullptr).IsInvalidArgument());
+ ASSERT_TRUE(r.ExtractDouble(r.root(), nullptr, nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractString(r.root(), nullptr, nullptr).IsInvalidArgument());
ASSERT_TRUE(r.ExtractObject(r.root(), nullptr, nullptr).IsInvalidArgument());
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/0a7940d6/src/kudu/util/jsonreader.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/jsonreader.cc b/src/kudu/util/jsonreader.cc
index 969d123..251c1e8 100644
--- a/src/kudu/util/jsonreader.cc
+++ b/src/kudu/util/jsonreader.cc
@@ -17,8 +17,12 @@
#include "kudu/util/jsonreader.h"
+#include <ostream>
#include <utility>
+#include <glog/logging.h>
+#include <rapidjson/rapidjson.h>
+
#include "kudu/gutil/port.h"
#include "kudu/gutil/strings/substitute.h"
@@ -29,6 +33,29 @@ using strings::Substitute;
namespace kudu {
+namespace {
+const char* const TypeToString(rapidjson::Type t) {
+ switch (t) {
+ case rapidjson::kNullType:
+ return "null";
+ case rapidjson::kFalseType:
+ return "false";
+ case rapidjson::kTrueType:
+ return "true";
+ case rapidjson::kObjectType:
+ return "object";
+ case rapidjson::kArrayType:
+ return "array";
+ case rapidjson::kStringType:
+ return "string";
+ case rapidjson::kNumberType:
+ return "number";
+ default:
+ LOG(FATAL) << "unexpected type: " << t;
+ }
+}
+} // anonymous namespace
+
JsonReader::JsonReader(string text) : text_(std::move(text)) {}
JsonReader::~JsonReader() {
@@ -49,8 +76,8 @@ Status JsonReader::ExtractBool(const Value* object,
RETURN_NOT_OK(ExtractField(object, field, &val));
if (PREDICT_FALSE(!val->IsBool())) {
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected bool but got $0",
- val->GetType()));
+ "wrong type during field extraction: expected bool but got $0",
+ TypeToString(val->GetType())));
}
*result = val->GetBool();
return Status::OK();
@@ -63,8 +90,8 @@ Status JsonReader::ExtractInt32(const Value* object,
RETURN_NOT_OK(ExtractField(object, field, &val));
if (PREDICT_FALSE(!val->IsInt())) {
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected int32 but got $0",
- val->GetType()));
+ "wrong type during field extraction: expected int32 but got $0",
+ TypeToString(val->GetType())));
}
*result = val->GetInt();
return Status::OK();
@@ -77,8 +104,8 @@ Status JsonReader::ExtractUint32(const Value* object,
RETURN_NOT_OK(ExtractField(object, field, &val));
if (PREDICT_FALSE(!val->IsUint())) {
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected uint32 but got $0",
- val->GetType()));
+ "wrong type during field extraction: expected uint32 but got $0",
+ TypeToString(val->GetType())));
}
*result = val->GetUint();
return Status::OK();
@@ -91,8 +118,8 @@ Status JsonReader::ExtractInt64(const Value* object,
RETURN_NOT_OK(ExtractField(object, field, &val));
if (PREDICT_FALSE(!val->IsInt64())) {
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected int64 but got $0",
- val->GetType())); }
+ "wrong type during field extraction: expected int64 but got $0",
+ TypeToString(val->GetType()))); }
*result = val->GetInt64();
return Status::OK();
}
@@ -104,12 +131,25 @@ Status JsonReader::ExtractUint64(const Value* object,
RETURN_NOT_OK(ExtractField(object, field, &val));
if (PREDICT_FALSE(!val->IsUint64())) {
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected uint64 but got $0",
- val->GetType())); }
+ "wrong type during field extraction: expected uint64 but got $0",
+ TypeToString(val->GetType()))); }
*result = val->GetUint64();
return Status::OK();
}
+Status JsonReader::ExtractDouble(const Value* object,
+ const char* field,
+ double* result) const {
+ const Value* val;
+ RETURN_NOT_OK(ExtractField(object, field, &val));
+ if (PREDICT_FALSE(!val->IsDouble())) {
+ return Status::InvalidArgument(Substitute(
+ "wrong type during field extraction: expected double but got $0",
+ TypeToString(val->GetType()))); }
+ *result = val->GetDouble();
+ return Status::OK();
+}
+
Status JsonReader::ExtractString(const Value* object,
const char* field,
string* result) const {
@@ -121,8 +161,8 @@ Status JsonReader::ExtractString(const Value* object,
return Status::OK();
}
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected string but got $0",
- val->GetType())); }
+ "wrong type during field extraction: expected string but got $0",
+ TypeToString(val->GetType()))); }
result->assign(val->GetString());
return Status::OK();
}
@@ -134,8 +174,8 @@ Status JsonReader::ExtractObject(const Value* object,
RETURN_NOT_OK(ExtractField(object, field, &val));
if (PREDICT_FALSE(!val->IsObject())) {
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected object but got $0",
- val->GetType())); }
+ "wrong type during field extraction: expected object but got $0",
+ TypeToString(val->GetType()))); }
*result = val;
return Status::OK();
}
@@ -147,8 +187,8 @@ Status JsonReader::ExtractObjectArray(const Value* object,
RETURN_NOT_OK(ExtractField(object, field, &val));
if (PREDICT_FALSE(!val->IsArray())) {
return Status::InvalidArgument(Substitute(
- "Wrong type during field extraction: expected object array but got $0",
- val->GetType())); }
+ "wrong type during field extraction: expected object array but got $0",
+ TypeToString(val->GetType()))); }
for (Value::ConstValueIterator iter = val->Begin(); iter != val->End(); ++iter) {
result->push_back(iter);
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/0a7940d6/src/kudu/util/jsonreader.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/jsonreader.h b/src/kudu/util/jsonreader.h
index 2dd3651..be2412b 100644
--- a/src/kudu/util/jsonreader.h
+++ b/src/kudu/util/jsonreader.h
@@ -72,6 +72,10 @@ class JsonReader {
const char* field,
std::string* result) const;
+ Status ExtractDouble(const rapidjson::Value* object,
+ const char* field,
+ double* result) const;
+
// 'result' is only valid for as long as JsonReader is alive.
Status ExtractObject(const rapidjson::Value* object,
const char* field,
[2/2] kudu git commit: Add missing test dependency for Hive tests
Posted by da...@apache.org.
Add missing test dependency for Hive tests
The HMS client internally calls into the JobConf class, which is
provided by the org.apache.hadoop.hadoop-mapreduce-client-core artifact.
The HMS only specifies an optional dependency on
hadoop-mapreduce-client-core, so we are forced to depend on it
explicitly.
Change-Id: I33187880247b3fe930ff521b0671345646e0fbf4
Reviewed-on: http://gerrit.cloudera.org:8080/11733
Reviewed-by: Hao Hao <ha...@cloudera.com>
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b3486d99
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b3486d99
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b3486d99
Branch: refs/heads/master
Commit: b3486d99a1e1bb5debef82912728d522f58621b7
Parents: 0a7940d
Author: Dan Burkert <da...@apache.org>
Authored: Thu Oct 18 17:18:32 2018 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Fri Oct 19 19:23:54 2018 +0000
----------------------------------------------------------------------
java/gradle/dependencies.gradle | 1 +
java/kudu-client/build.gradle | 4 ++++
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/b3486d99/java/gradle/dependencies.gradle
----------------------------------------------------------------------
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index 267e452..52a5079 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -83,6 +83,7 @@ libs += [
hadoopClient : "org.apache.hadoop:hadoop-client:$versions.hadoop",
hadoopCommon : "org.apache.hadoop:hadoop-common:$versions.hadoop",
hadoopMRClientCommon : "org.apache.hadoop:hadoop-mapreduce-client-common:$versions.hadoop",
+ hadoopMRClientCore : "org.apache.hadoop:hadoop-mapreduce-client-core:$versions.hadoop",
hamcrestCore : "org.hamcrest:hamcrest-core:$versions.hamcrest",
hiveMetastore : "org.apache.hive:hive-metastore:$versions.hive",
hiveMetastoreTest : "org.apache.hive:hive-metastore:$versions.hive:tests",
http://git-wip-us.apache.org/repos/asf/kudu/blob/b3486d99/java/kudu-client/build.gradle
----------------------------------------------------------------------
diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle
index 363135b..2b99355 100644
--- a/java/kudu-client/build.gradle
+++ b/java/kudu-client/build.gradle
@@ -38,6 +38,10 @@ dependencies {
testCompile project(":kudu-test-utils")
testCompile libs.hamcrestCore
testCompile libs.hiveMetastore
+ // The HMS client relies on the MR client-core artifact for JobConf, but only
+ // specifies it as an optional dependency. Gradle doesn't pull in optional
+ // dependencies, so we have to depend on it directly.
+ testCompile libs.hadoopMRClientCore
testCompile libs.junit
testCompile libs.log4j
testCompile libs.mockitoCore