You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2019/03/07 18:58:41 UTC
[kudu] 02/02: [thirdparty] Bump RapidJSON version to 1.1.0
This is an automated email from the ASF dual-hosted git repository.
adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit e0557a26843085a7264800ec2d5ff92e125375c0
Author: Yingchun Lai <40...@qq.com>
AuthorDate: Fri Mar 1 10:49:11 2019 -0500
[thirdparty] Bump RapidJSON version to 1.1.0
Compatibility fix:
* Document::GetParseError now returns ParseErrorCode instead of a string.
We should wrap it with GetParseError_En(). The error messages are also
changed. Strings in unit tests should be changed correspondingly.
Tests:
* Pass all existing tests
Change-Id: I1c5e6bf6b8c6802bde28716487abb2b242a7a578
Reviewed-on: http://gerrit.cloudera.org:8080/12643
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Reviewed-by: Mike Percy <mp...@apache.org>
---
src/kudu/master/hms_notification_log_listener.cc | 4 +-
src/kudu/tablet/tablet-test.cc | 14 +-
src/kudu/tools/kudu-tool-test.cc | 2 +-
src/kudu/util/jsonreader-test.cc | 9 +-
src/kudu/util/jsonreader.cc | 4 +-
src/kudu/util/jsonwriter-test.cc | 8 +-
src/kudu/util/jsonwriter.cc | 11 +-
src/kudu/util/jsonwriter.h | 2 +-
thirdparty/download-thirdparty.sh | 5 +-
...json-fix-signed-unsigned-conversion-error.patch | 218 +++++++++++++++++++++
thirdparty/vars.sh | 2 +-
11 files changed, 252 insertions(+), 27 deletions(-)
diff --git a/src/kudu/master/hms_notification_log_listener.cc b/src/kudu/master/hms_notification_log_listener.cc
index 0c751a6..64a8d14 100644
--- a/src/kudu/master/hms_notification_log_listener.cc
+++ b/src/kudu/master/hms_notification_log_listener.cc
@@ -28,6 +28,7 @@
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <rapidjson/document.h>
+#include <rapidjson/error/en.h>
#include "kudu/gutil/callback.h"
#include "kudu/gutil/macros.h"
@@ -189,7 +190,8 @@ Status ParseMessage(const hive::NotificationEvent& event, Document* message) {
return Status::NotSupported("unknown message format", event.messageFormat);
}
if (message->Parse<0>(event.message.c_str()).HasParseError()) {
- return Status::Corruption("failed to parse message", message->GetParseError());
+ return Status::Corruption("failed to parse message",
+ rapidjson::GetParseError_En(message->GetParseError()));
}
return Status::OK();
}
diff --git a/src/kudu/tablet/tablet-test.cc b/src/kudu/tablet/tablet-test.cc
index 457a51a..3ceb6c1 100644
--- a/src/kudu/tablet/tablet-test.cc
+++ b/src/kudu/tablet/tablet-test.cc
@@ -1104,12 +1104,18 @@ TYPED_TEST(TestTablet, TestMetricsInit) {
// Create a tablet, but do not open it
this->CreateTestTablet();
MetricRegistry* registry = this->harness()->metrics_registry();
- std::ostringstream out;
- JsonWriter writer(&out, JsonWriter::PRETTY);
- ASSERT_OK(registry->WriteAsJson(&writer, { "*" }, MetricJsonOptions()));
+ {
+ std::ostringstream out;
+ JsonWriter writer(&out, JsonWriter::PRETTY);
+ ASSERT_OK(registry->WriteAsJson(&writer, { "*" }, MetricJsonOptions()));
+ }
// Open tablet, should still work
this->harness()->Open();
- ASSERT_OK(registry->WriteAsJson(&writer, { "*" }, MetricJsonOptions()));
+ {
+ std::ostringstream out;
+ JsonWriter writer(&out, JsonWriter::PRETTY);
+ ASSERT_OK(registry->WriteAsJson(&writer, { "*" }, MetricJsonOptions()));
+ }
}
// Test that we find the correct log segment size for different indexes.
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 6e3909e..615950b 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -4011,7 +4011,7 @@ TEST_F(ToolTest, TestParseStacks) {
&stderr);
ASSERT_TRUE(s.IsRuntimeError());
ASSERT_STR_MATCHES(stderr, "failed to parse stacks from .*: at line 1: "
- "invalid JSON payload.*lacks ending quotation");
+ "invalid JSON payload.*Missing a closing quotation mark in string");
}
class Is343ReplicaUtilTest :
diff --git a/src/kudu/util/jsonreader-test.cc b/src/kudu/util/jsonreader-test.cc
index 250091f..b321538 100644
--- a/src/kudu/util/jsonreader-test.cc
+++ b/src/kudu/util/jsonreader-test.cc
@@ -43,7 +43,7 @@ TEST(JsonReaderTest, Corrupt) {
Status s = r.Init();
ASSERT_TRUE(s.IsCorruption());
ASSERT_STR_CONTAINS(
- s.ToString(), "JSON text is corrupt: Text only contains white space(s)");
+ s.ToString(), "JSON text is corrupt: The document is empty.");
}
TEST(JsonReaderTest, Empty) {
@@ -144,13 +144,6 @@ TEST(JsonReaderTest, LessBasic) {
}
TEST(JsonReaderTest, SignedAndUnsignedInts) {
- // The rapidjson code has some improper handling of the min int32 and min
- // int64 that exposes UB.
- #if defined(ADDRESS_SANITIZER)
- LOG(WARNING) << "this test is skipped in ASAN builds";
- return;
- #endif
-
constexpr auto kMaxInt32 = std::numeric_limits<int32_t>::max();
constexpr auto kMaxInt64 = std::numeric_limits<int64_t>::max();
constexpr auto kMaxUint32 = std::numeric_limits<uint32_t>::max();
diff --git a/src/kudu/util/jsonreader.cc b/src/kudu/util/jsonreader.cc
index 251c1e8..05cde80 100644
--- a/src/kudu/util/jsonreader.cc
+++ b/src/kudu/util/jsonreader.cc
@@ -21,6 +21,7 @@
#include <utility>
#include <glog/logging.h>
+#include <rapidjson/error/en.h>
#include <rapidjson/rapidjson.h>
#include "kudu/gutil/port.h"
@@ -64,7 +65,8 @@ JsonReader::~JsonReader() {
Status JsonReader::Init() {
document_.Parse<0>(text_.c_str());
if (document_.HasParseError()) {
- return Status::Corruption("JSON text is corrupt", document_.GetParseError());
+ return Status::Corruption("JSON text is corrupt",
+ rapidjson::GetParseError_En(document_.GetParseError()));
}
return Status::OK();
}
diff --git a/src/kudu/util/jsonwriter-test.cc b/src/kudu/util/jsonwriter-test.cc
index 6f9b10d..b9a1fdb 100644
--- a/src/kudu/util/jsonwriter-test.cc
+++ b/src/kudu/util/jsonwriter-test.cc
@@ -83,8 +83,8 @@ TEST_F(TestJsonWriter, TestPBAllFieldTypes) {
" \"optional_fixed64\": 8,\n"
" \"optional_sfixed32\": 9,\n"
" \"optional_sfixed64\": 10,\n"
- " \"optional_float\": 11,\n"
- " \"optional_double\": 12,\n"
+ " \"optional_float\": 11.0,\n"
+ " \"optional_double\": 12.0,\n"
" \"optional_bool\": true,\n"
" \"optional_string\": \"hello world\",\n"
" \"optional_redacted_string\": \"<redacted>\",\n"
@@ -101,8 +101,8 @@ TEST_F(TestJsonWriter, TestPBAllFieldTypes) {
"\"optional_fixed64\":8,"
"\"optional_sfixed32\":9,"
"\"optional_sfixed64\":10,"
- "\"optional_float\":11,"
- "\"optional_double\":12,"
+ "\"optional_float\":11.0,"
+ "\"optional_double\":12.0,"
"\"optional_bool\":true,"
"\"optional_string\":\"hello world\","
"\"optional_redacted_string\":\"<redacted>\","
diff --git a/src/kudu/util/jsonwriter.cc b/src/kudu/util/jsonwriter.cc
index 3a5580c..bfb047c 100644
--- a/src/kudu/util/jsonwriter.cc
+++ b/src/kudu/util/jsonwriter.cc
@@ -26,9 +26,11 @@
#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/message.h>
-#include <rapidjson/writer.h>
+#include <rapidjson/encodings.h>
+// IWYU pragma: no_include <rapidjson/internal/../rapidjson.h>
#include <rapidjson/prettywriter.h>
-#include <rapidjson/rapidjson.h>
+#include <rapidjson/rapidjson.h> // IWYU pragma: keep
+#include <rapidjson/writer.h>
#include "kudu/gutil/port.h"
#include "kudu/util/faststring.h"
@@ -49,10 +51,11 @@ namespace kudu {
// Since Squeasel exposes a stringstream as its interface, this is needed to avoid overcopying.
class UTF8StringStreamBuffer {
public:
+ typedef typename rapidjson::UTF8<>::Ch Ch;
explicit UTF8StringStreamBuffer(std::ostringstream* out);
~UTF8StringStreamBuffer();
- void Put(rapidjson::UTF8<>::Ch c);
+ void Put(Ch c);
void Flush();
private:
@@ -296,7 +299,7 @@ UTF8StringStreamBuffer::~UTF8StringStreamBuffer() {
DCHECK_EQ(buf_.size(), 0) << "Forgot to flush!";
}
-void UTF8StringStreamBuffer::Put(rapidjson::UTF8<>::Ch c) {
+void UTF8StringStreamBuffer::Put(Ch c) {
buf_.push_back(c);
}
diff --git a/src/kudu/util/jsonwriter.h b/src/kudu/util/jsonwriter.h
index 24b4575..9b4abf0 100644
--- a/src/kudu/util/jsonwriter.h
+++ b/src/kudu/util/jsonwriter.h
@@ -70,7 +70,7 @@ class JsonWriter {
// Convert the given protobuf message to JSON.
// The output respects redaction for 'string' and 'bytes' fields.
- void Protobuf(const google::protobuf::Message& message);
+ void Protobuf(const google::protobuf::Message& pb);
template<typename T>
void Value(const T& val);
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 95a3aa1..a6fa2b9 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -258,11 +258,12 @@ fetch_and_patch \
$LIBEV_SOURCE \
$LIBEV_PATCHLEVEL
-RAPIDJSON_PATCHLEVEL=0
+RAPIDJSON_PATCHLEVEL=1
fetch_and_patch \
rapidjson-${RAPIDJSON_VERSION}.zip \
$RAPIDJSON_SOURCE \
- $RAPIDJSON_PATCHLEVEL
+ $RAPIDJSON_PATCHLEVEL \
+ "patch -p1 < $TP_DIR/patches/rapidjson-fix-signed-unsigned-conversion-error.patch"
SQUEASEL_PATCHLEVEL=0
fetch_and_patch \
diff --git a/thirdparty/patches/rapidjson-fix-signed-unsigned-conversion-error.patch b/thirdparty/patches/rapidjson-fix-signed-unsigned-conversion-error.patch
new file mode 100644
index 0000000..562f2ce
--- /dev/null
+++ b/thirdparty/patches/rapidjson-fix-signed-unsigned-conversion-error.patch
@@ -0,0 +1,218 @@
+commit 3cc77d5d635c2411f327cc4f262f37abb66ff43c
+Author: Zhihao Yuan <li...@gmail.com>
+Date: Wed Jan 18 16:16:07 2017 -0600
+
+ Treat signed-unsigned conversions as errors.
+
+diff --git a/include/rapidjson/encodedstream.h b/include/rapidjson/encodedstream.h
+index 14506838..223601c0 100644
+--- a/include/rapidjson/encodedstream.h
++++ b/include/rapidjson/encodedstream.h
+@@ -200,7 +200,7 @@ private:
+ // xx xx xx xx UTF-8
+
+ if (!hasBOM_) {
+- unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0);
++ int pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0);
+ switch (pattern) {
+ case 0x08: type_ = kUTF32BE; break;
+ case 0x0A: type_ = kUTF16BE; break;
+diff --git a/include/rapidjson/encodings.h b/include/rapidjson/encodings.h
+index baa7c2b1..ed7d44d3 100644
+--- a/include/rapidjson/encodings.h
++++ b/include/rapidjson/encodings.h
+@@ -157,7 +157,7 @@ struct UTF8 {
+ if (type >= 32) {
+ *codepoint = 0;
+ } else {
+- *codepoint = (0xFF >> type) & static_cast<unsigned char>(c);
++ *codepoint = (0xFFu >> type) & static_cast<unsigned char>(c);
+ }
+ bool result = true;
+ switch (type) {
+@@ -283,7 +283,7 @@ struct UTF16 {
+ RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+ unsigned v = codepoint - 0x10000;
+ os.Put(static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800));
+- os.Put((v & 0x3FF) | 0xDC00);
++ os.Put(static_cast<typename OutputStream::Ch>((v & 0x3FF) | 0xDC00));
+ }
+ }
+
+@@ -299,7 +299,7 @@ struct UTF16 {
+ RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+ unsigned v = codepoint - 0x10000;
+ PutUnsafe(os, static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800));
+- PutUnsafe(os, (v & 0x3FF) | 0xDC00);
++ PutUnsafe(os, static_cast<typename OutputStream::Ch>((v & 0x3FF) | 0xDC00));
+ }
+ }
+
+diff --git a/include/rapidjson/internal/dtoa.h b/include/rapidjson/internal/dtoa.h
+index 8d6350e6..bf2e9b2e 100644
+--- a/include/rapidjson/internal/dtoa.h
++++ b/include/rapidjson/internal/dtoa.h
+@@ -41,7 +41,7 @@ inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uin
+ }
+ }
+
+-inline unsigned CountDecimalDigit32(uint32_t n) {
++inline int CountDecimalDigit32(uint32_t n) {
+ // Simple pure C++ implementation was faster than __builtin_clz version in this situation.
+ if (n < 10) return 1;
+ if (n < 100) return 2;
+@@ -63,7 +63,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff
+ const DiyFp wp_w = Mp - W;
+ uint32_t p1 = static_cast<uint32_t>(Mp.f >> -one.e);
+ uint64_t p2 = Mp.f & (one.f - 1);
+- unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9]
++ int kappa = CountDecimalDigit32(p1); // kappa in [0, 9]
+ *len = 0;
+
+ while (kappa > 0) {
+@@ -102,8 +102,8 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff
+ kappa--;
+ if (p2 < delta) {
+ *K += kappa;
+- int index = -static_cast<int>(kappa);
+- GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast<int>(kappa)] : 0));
++ int index = -kappa;
++ GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0));
+ return;
+ }
+ }
+diff --git a/include/rapidjson/internal/ieee754.h b/include/rapidjson/internal/ieee754.h
+index 82bb0b99..c2684ba2 100644
+--- a/include/rapidjson/internal/ieee754.h
++++ b/include/rapidjson/internal/ieee754.h
+@@ -48,13 +48,13 @@ public:
+ int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; }
+ uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; }
+
+- static unsigned EffectiveSignificandSize(int order) {
++ static int EffectiveSignificandSize(int order) {
+ if (order >= -1021)
+ return 53;
+ else if (order <= -1074)
+ return 0;
+ else
+- return static_cast<unsigned>(order) + 1074;
++ return order + 1074;
+ }
+
+ private:
+diff --git a/include/rapidjson/internal/regex.h b/include/rapidjson/internal/regex.h
+index 936b7144..1369ea26 100644
+--- a/include/rapidjson/internal/regex.h
++++ b/include/rapidjson/internal/regex.h
+@@ -688,8 +688,8 @@ private:
+ bool matched = AddState(l, s.out);
+ return AddState(l, s.out1) || matched;
+ }
+- else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) {
+- stateSet_[index >> 5] |= (1 << (index & 31));
++ else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) {
++ stateSet_[index >> 5] |= (1u << (index & 31));
+ *l.template PushUnsafe<SizeType>() = index;
+ }
+ return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation.
+diff --git a/include/rapidjson/internal/strtod.h b/include/rapidjson/internal/strtod.h
+index 289c413b..adf49e34 100644
+--- a/include/rapidjson/internal/strtod.h
++++ b/include/rapidjson/internal/strtod.h
+@@ -140,8 +140,8 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit
+ significand++;
+
+ size_t remaining = length - i;
+- const unsigned kUlpShift = 3;
+- const unsigned kUlp = 1 << kUlpShift;
++ const int kUlpShift = 3;
++ const int kUlp = 1 << kUlpShift;
+ int64_t error = (remaining == 0) ? 0 : kUlp / 2;
+
+ DiyFp v(significand, 0);
+@@ -177,17 +177,17 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit
+ v = v.Normalize();
+ error <<= oldExp - v.e;
+
+- const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e);
+- unsigned precisionSize = 64 - effectiveSignificandSize;
++ const int effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e);
++ int precisionSize = 64 - effectiveSignificandSize;
+ if (precisionSize + kUlpShift >= 64) {
+- unsigned scaleExp = (precisionSize + kUlpShift) - 63;
++ int scaleExp = (precisionSize + kUlpShift) - 63;
+ v.f >>= scaleExp;
+ v.e += scaleExp;
+- error = (error >> scaleExp) + 1 + static_cast<int>(kUlp);
++ error = (error >> scaleExp) + 1 + kUlp;
+ precisionSize -= scaleExp;
+ }
+
+- DiyFp rounded(v.f >> precisionSize, v.e + static_cast<int>(precisionSize));
++ DiyFp rounded(v.f >> precisionSize, v.e + precisionSize);
+ const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp;
+ const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp;
+ if (precisionBits >= halfWay + static_cast<unsigned>(error)) {
+diff --git a/include/rapidjson/istreamwrapper.h b/include/rapidjson/istreamwrapper.h
+index f5fe2897..8639c8c3 100644
+--- a/include/rapidjson/istreamwrapper.h
++++ b/include/rapidjson/istreamwrapper.h
+@@ -54,7 +54,7 @@ public:
+
+ Ch Peek() const {
+ typename StreamType::int_type c = stream_.peek();
+- return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast<Ch>(c) : '\0';
++ return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast<Ch>(c) : static_cast<Ch>('\0');
+ }
+
+ Ch Take() {
+diff --git a/include/rapidjson/pointer.h b/include/rapidjson/pointer.h
+index 4d6391f9..bc7acfd0 100644
+--- a/include/rapidjson/pointer.h
++++ b/include/rapidjson/pointer.h
+@@ -274,7 +274,7 @@ public:
+ else {
+ Ch name[21];
+ for (size_t i = 0; i <= length; i++)
+- name[i] = buffer[i];
++ name[i] = static_cast<Ch>(buffer[i]);
+ Token token = { name, length, index };
+ return Append(token, allocator);
+ }
+@@ -1029,8 +1029,8 @@ private:
+ unsigned char u = static_cast<unsigned char>(c);
+ static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
+ os_.Put('%');
+- os_.Put(hexDigits[u >> 4]);
+- os_.Put(hexDigits[u & 15]);
++ os_.Put(static_cast<typename OutputStream::Ch>(hexDigits[u >> 4]));
++ os_.Put(static_cast<typename OutputStream::Ch>(hexDigits[u & 15]));
+ }
+ private:
+ OutputStream& os_;
+diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h
+index 4760d1b4..3f81d9bb 100644
+--- a/include/rapidjson/schema.h
++++ b/include/rapidjson/schema.h
+@@ -1263,7 +1263,7 @@ struct TokenHelper {
+ char buffer[21];
+ size_t length = static_cast<size_t>((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer);
+ for (size_t i = 0; i < length; i++)
+- *documentStack.template Push<Ch>() = buffer[i];
++ *documentStack.template Push<Ch>() = static_cast<Ch>(buffer[i]);
+ }
+ };
+
+diff --git a/include/rapidjson/writer.h b/include/rapidjson/writer.h
+index 8f6e174f..874c555c 100644
+--- a/include/rapidjson/writer.h
++++ b/include/rapidjson/writer.h
+@@ -352,7 +352,7 @@ protected:
+ char* end = internal::dtoa(d, buffer, maxDecimalPlaces_);
+ PutReserve(*os_, static_cast<size_t>(end - buffer));
+ for (char* p = buffer; p != end; ++p)
+- PutUnsafe(*os_, static_cast<typename TargetEncoding::Ch>(*p));
++ PutUnsafe(*os_, static_cast<typename OutputStream::Ch>(*p));
+ return true;
+ }
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index fe4cd10..1c142a7 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -86,7 +86,7 @@ LIBEV_VERSION=4.20
LIBEV_NAME=libev-$LIBEV_VERSION
LIBEV_SOURCE=$TP_SOURCE_DIR/$LIBEV_NAME
-RAPIDJSON_VERSION=0.11
+RAPIDJSON_VERSION=1.1.0
RAPIDJSON_NAME=rapidjson-$RAPIDJSON_VERSION
RAPIDJSON_SOURCE=$TP_SOURCE_DIR/$RAPIDJSON_NAME