You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by th...@apache.org on 2016/04/13 07:29:32 UTC
avro git commit: AVRO-1784. C++: Should have a json encoder that
pretty prints
Repository: avro
Updated Branches:
refs/heads/AVRO-1784 [created] e98caa993
AVRO-1784. C++: Should have a json encoder that pretty prints
Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/e98caa99
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/e98caa99
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/e98caa99
Branch: refs/heads/AVRO-1784
Commit: e98caa9937c47a4b44c80e0497995fd404102755
Parents: 32067de
Author: Thiruvalluvan M. G <th...@verticloud.com>
Authored: Wed Apr 13 10:53:29 2016 +0530
Committer: Thiruvalluvan M. G <th...@verticloud.com>
Committed: Wed Apr 13 10:58:22 2016 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 +
lang/c++/api/Encoder.hh | 7 ++-
lang/c++/impl/json/JsonDom.cc | 4 +-
lang/c++/impl/json/JsonDom.hh | 5 +-
lang/c++/impl/json/JsonIO.cc | 19 -------
lang/c++/impl/json/JsonIO.hh | 78 +++++++++++++++++++++++++-
lang/c++/impl/parsing/JsonCodec.cc | 97 ++++++++++++++++++---------------
lang/c++/test/CodecTests.cc | 14 ++++-
8 files changed, 154 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b44745f..1399a94 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -89,6 +89,8 @@ Avro 1.8.0 (22 January 2016)
AVRO-1767. JS: Fall back to unwrapped union values. (Matthieu Monsch via blue)
+ AVRO-1784. C++: Should have a json encoder that pretty prints. (John McClean via thiru)
+
OPTIMIZATIONS
AVRO-1760. Java: Thread scalability problem with the use of SynchronizedMap
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/api/Encoder.hh
----------------------------------------------------------------------
diff --git a/lang/c++/api/Encoder.hh b/lang/c++/api/Encoder.hh
index a5dce71..e13fa69 100644
--- a/lang/c++/api/Encoder.hh
+++ b/lang/c++/api/Encoder.hh
@@ -156,10 +156,15 @@ AVRO_DECL EncoderPtr validatingEncoder(const ValidSchema& schema,
const EncoderPtr& base);
/**
- * Returns an encoder that can encode Avro standard for JSON.
+ * Returns an encoder that encodes Avro standard for JSON.
*/
AVRO_DECL EncoderPtr jsonEncoder(const ValidSchema& schema);
+/**
+ * Returns an encoder that encodes Avro standard for pretty printed JSON.
+ */
+AVRO_DECL EncoderPtr jsonPrettyEncoder(const ValidSchema& schema);
+
} // namespace avro
#endif
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonDom.cc
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonDom.cc b/lang/c++/impl/json/JsonDom.cc
index 4c70b32..3f52f36 100644
--- a/lang/c++/impl/json/JsonDom.cc
+++ b/lang/c++/impl/json/JsonDom.cc
@@ -110,7 +110,7 @@ Entity loadEntity(const uint8_t* text, size_t len)
return loadEntity(*in);
}
-void writeEntity(JsonGenerator& g, const Entity& n)
+void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n)
{
switch (n.type()) {
case etNull:
@@ -166,7 +166,7 @@ void Entity::ensureType(EntityType type) const
std::string Entity::toString() const
{
std::auto_ptr<OutputStream> out = memoryOutputStream();
- JsonGenerator g;
+ JsonGenerator<JsonNullFormatter> g;
g.init(*out);
writeEntity(g, *this);
g.flush();
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonDom.hh
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonDom.hh b/lang/c++/impl/json/JsonDom.hh
index ae99d4f..7de4fbc 100644
--- a/lang/c++/impl/json/JsonDom.hh
+++ b/lang/c++/impl/json/JsonDom.hh
@@ -44,6 +44,9 @@ typedef std::vector<Entity> Array;
typedef std::map<std::string, Entity> Object;
class AVRO_DECL JsonParser;
+class JsonNullFormatter;
+
+template<typename F = JsonNullFormatter>
class AVRO_DECL JsonGenerator;
enum EntityType {
@@ -144,7 +147,7 @@ AVRO_DECL Entity loadEntity(InputStream& in);
AVRO_DECL Entity loadEntity(const char* text);
AVRO_DECL Entity loadEntity(const uint8_t* text, size_t len);
-void writeEntity(JsonGenerator& g, const Entity& n);
+void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n);
}
}
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonIO.cc
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonIO.cc b/lang/c++/impl/json/JsonIO.cc
index f21e822..2e7d82f 100644
--- a/lang/c++/impl/json/JsonIO.cc
+++ b/lang/c++/impl/json/JsonIO.cc
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-#include "boost/math/special_functions/fpclassify.hpp"
#include "JsonIO.hh"
namespace avro {
@@ -351,24 +350,6 @@ JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n, Token tk)
return tk;
}
-void JsonGenerator::encodeNumber(double t) {
- sep();
- std::ostringstream oss;
- if (boost::math::isfinite(t)) {
- oss << t;
- } else if (boost::math::isnan(t)) {
- oss << "NaN";
- } else if (t == std::numeric_limits<double>::infinity()) {
- oss << "Infinity";
- } else {
- oss << "-Infinity";
- }
- const std::string& s = oss.str();
- out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size());
- sep2();
-}
-
-
}
}
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonIO.hh
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonIO.hh b/lang/c++/impl/json/JsonIO.hh
index 1406d7a..a5ada2d 100644
--- a/lang/c++/impl/json/JsonIO.hh
+++ b/lang/c++/impl/json/JsonIO.hh
@@ -23,6 +23,7 @@
#include <stack>
#include <string>
#include <sstream>
+#include <boost/math/special_functions/fpclassify.hpp>
#include <boost/utility.hpp>
#include "Config.hh"
@@ -132,8 +133,58 @@ public:
}
};
+struct AVRO_DECL JsonNullFormatter {
+ JsonNullFormatter(StreamWriter&) { }
+
+ void handleObjectStart() {}
+ void handleObjectEnd() {}
+ void handleValueEnd() {}
+ void handleColon() {}
+};
+
+class AVRO_DECL JsonPrettyFormatter {
+ StreamWriter& out_;
+ size_t level_;
+ std::vector<uint8_t> indent_;
+
+ static const int CHARS_PER_LEVEL = 2;
+
+ void printIndent() {
+ size_t charsToIndent = level_ * CHARS_PER_LEVEL;
+ if (indent_.size() < charsToIndent) {
+ indent_.resize(charsToIndent * 2, ' ');
+ }
+ out_.writeBytes(&(indent_[0]), charsToIndent);
+ }
+public:
+ JsonPrettyFormatter(StreamWriter& out) : out_(out), level_(0), indent_(10, ' ') { }
+
+ void handleObjectStart() {
+ out_.write('\n');
+ ++level_;
+ printIndent();
+ }
+
+ void handleObjectEnd() {
+ out_.write('\n');
+ --level_;
+ printIndent();
+ }
+
+ void handleValueEnd() {
+ out_.write('\n');
+ printIndent();
+ }
+
+ void handleColon() {
+ out_.write(' ');
+ }
+};
+
+template <class F>
class AVRO_DECL JsonGenerator {
StreamWriter out_;
+ F formatter_;
enum State {
stStart,
stArray0,
@@ -210,6 +261,7 @@ class AVRO_DECL JsonGenerator {
void sep() {
if (top == stArrayN) {
out_.write(',');
+ formatter_.handleValueEnd();
} else if (top == stArray0) {
top = stArrayN;
}
@@ -222,7 +274,7 @@ class AVRO_DECL JsonGenerator {
}
public:
- JsonGenerator() : top(stStart) { }
+ JsonGenerator() : formatter_(out_), top(stStart) { }
void init(OutputStream& os) {
out_.reset(os);
@@ -258,13 +310,30 @@ public:
sep2();
}
- void encodeNumber(double t);
+ void encodeNumber(double t) {
+ sep();
+ std::ostringstream oss;
+ if (boost::math::isfinite(t)) {
+ oss << t;
+ } else if (boost::math::isnan(t)) {
+ oss << "NaN";
+ } else if (t == std::numeric_limits<double>::infinity()) {
+ oss << "Infinity";
+ } else {
+ oss << "-Infinity";
+ }
+ const std::string& s = oss.str();
+ out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size());
+ sep2();
+ }
+
void encodeString(const std::string& s) {
if (top == stMap0) {
top = stKey;
} else if (top == stMapN) {
out_.write(',');
+ formatter_.handleValueEnd();
top = stKey;
} else if (top == stKey) {
top = stMapN;
@@ -274,6 +343,7 @@ public:
doEncodeString(s);
if (top == stKey) {
out_.write(':');
+ formatter_.handleColon();
}
}
@@ -293,11 +363,13 @@ public:
stateStack.push(top);
top = stArray0;
out_.write('[');
+ formatter_.handleObjectStart();
}
void arrayEnd() {
top = stateStack.top();
stateStack.pop();
+ formatter_.handleObjectEnd();
out_.write(']');
sep2();
}
@@ -307,11 +379,13 @@ public:
stateStack.push(top);
top = stMap0;
out_.write('{');
+ formatter_.handleObjectStart();
}
void objectEnd() {
top = stateStack.top();
stateStack.pop();
+ formatter_.handleObjectEnd();
out_.write('}');
sep2();
}
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/parsing/JsonCodec.cc
----------------------------------------------------------------------
diff --git a/lang/c++/impl/parsing/JsonCodec.cc b/lang/c++/impl/parsing/JsonCodec.cc
index f18a1c9..ead44ca 100644
--- a/lang/c++/impl/parsing/JsonCodec.cc
+++ b/lang/c++/impl/parsing/JsonCodec.cc
@@ -54,6 +54,7 @@ using std::istringstream;
using avro::json::JsonParser;
using avro::json::JsonGenerator;
+using avro::json::JsonNullFormatter;
class JsonGrammarGenerator : public ValidatingGrammarGenerator {
ProductionPtr doGenerate(const NodePtr& n,
@@ -464,11 +465,11 @@ size_t JsonDecoder<P>::decodeUnionIndex()
return result;
}
-
+template<typename F = JsonNullFormatter>
class JsonHandler {
- JsonGenerator& generator_;
+ JsonGenerator<F>& generator_;
public:
- JsonHandler(JsonGenerator& g) : generator_(g) { }
+ JsonHandler(JsonGenerator<F>& g) : generator_(g) { }
size_t handle(const Symbol& s) {
switch (s.kind()) {
case Symbol::sRecordStart:
@@ -487,10 +488,10 @@ public:
}
};
-template <typename P>
+template <typename P, typename F = JsonNullFormatter>
class JsonEncoder : public Encoder {
- JsonGenerator out_;
- JsonHandler handler_;
+ JsonGenerator<F> out_;
+ JsonHandler<F> handler_;
P parser_;
void init(OutputStream& os);
@@ -518,49 +519,49 @@ public:
parser_(JsonGrammarGenerator().generate(schema), NULL, handler_) { }
};
-template<typename P>
-void JsonEncoder<P>::init(OutputStream& os)
+template<typename P, typename F>
+void JsonEncoder<P, F>::init(OutputStream& os)
{
out_.init(os);
}
-template<typename P>
-void JsonEncoder<P>::flush()
+template<typename P, typename F>
+void JsonEncoder<P, F>::flush()
{
parser_.processImplicitActions();
out_.flush();
}
-template<typename P>
-void JsonEncoder<P>::encodeNull()
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeNull()
{
parser_.advance(Symbol::sNull);
out_.encodeNull();
}
-template<typename P>
-void JsonEncoder<P>::encodeBool(bool b)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeBool(bool b)
{
parser_.advance(Symbol::sBool);
out_.encodeBool(b);
}
-template<typename P>
-void JsonEncoder<P>::encodeInt(int32_t i)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeInt(int32_t i)
{
parser_.advance(Symbol::sInt);
out_.encodeNumber(i);
}
-template<typename P>
-void JsonEncoder<P>::encodeLong(int64_t l)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeLong(int64_t l)
{
parser_.advance(Symbol::sLong);
out_.encodeNumber(l);
}
-template<typename P>
-void JsonEncoder<P>::encodeFloat(float f)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeFloat(float f)
{
parser_.advance(Symbol::sFloat);
if (f == std::numeric_limits<float>::infinity()) {
@@ -574,8 +575,8 @@ void JsonEncoder<P>::encodeFloat(float f)
}
}
-template<typename P>
-void JsonEncoder<P>::encodeDouble(double d)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeDouble(double d)
{
parser_.advance(Symbol::sDouble);
if (d == std::numeric_limits<double>::infinity()) {
@@ -589,74 +590,74 @@ void JsonEncoder<P>::encodeDouble(double d)
}
}
-template<typename P>
-void JsonEncoder<P>::encodeString(const std::string& s)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeString(const std::string& s)
{
parser_.advance(Symbol::sString);
out_.encodeString(s);
}
-template<typename P>
-void JsonEncoder<P>::encodeBytes(const uint8_t *bytes, size_t len)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeBytes(const uint8_t *bytes, size_t len)
{
parser_.advance(Symbol::sBytes);
out_.encodeBinary(bytes, len);
}
-template<typename P>
-void JsonEncoder<P>::encodeFixed(const uint8_t *bytes, size_t len)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeFixed(const uint8_t *bytes, size_t len)
{
parser_.advance(Symbol::sFixed);
parser_.assertSize(len);
out_.encodeBinary(bytes, len);
}
-template<typename P>
-void JsonEncoder<P>::encodeEnum(size_t e)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeEnum(size_t e)
{
parser_.advance(Symbol::sEnum);
const string& s = parser_.nameForIndex(e);
out_.encodeString(s);
}
-template<typename P>
-void JsonEncoder<P>::arrayStart()
+template<typename P, typename F>
+void JsonEncoder<P, F>::arrayStart()
{
parser_.advance(Symbol::sArrayStart);
out_.arrayStart();
}
-template<typename P>
-void JsonEncoder<P>::arrayEnd()
+template<typename P, typename F>
+void JsonEncoder<P, F>::arrayEnd()
{
parser_.popRepeater();
parser_.advance(Symbol::sArrayEnd);
out_.arrayEnd();
}
-template<typename P>
-void JsonEncoder<P>::mapStart()
+template<typename P, typename F>
+void JsonEncoder<P, F>::mapStart()
{
parser_.advance(Symbol::sMapStart);
out_.objectStart();
}
-template<typename P>
-void JsonEncoder<P>::mapEnd()
+template<typename P, typename F>
+void JsonEncoder<P, F>::mapEnd()
{
parser_.popRepeater();
parser_.advance(Symbol::sMapEnd);
out_.objectEnd();
}
-template<typename P>
-void JsonEncoder<P>::setItemCount(size_t count)
+template<typename P, typename F>
+void JsonEncoder<P, F>::setItemCount(size_t count)
{
parser_.setRepeatCount(count);
}
-template<typename P>
-void JsonEncoder<P>::startItem()
+template<typename P, typename F>
+void JsonEncoder<P, F>::startItem()
{
parser_.processImplicitActions();
if (parser_.top() != Symbol::sRepeater) {
@@ -664,8 +665,8 @@ void JsonEncoder<P>::startItem()
}
}
-template<typename P>
-void JsonEncoder<P>::encodeUnionIndex(size_t e)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeUnionIndex(size_t e)
{
parser_.advance(Symbol::sUnion);
@@ -689,7 +690,13 @@ DecoderPtr jsonDecoder(const ValidSchema& s)
EncoderPtr jsonEncoder(const ValidSchema& schema)
{
return boost::make_shared<parsing::JsonEncoder<
- parsing::SimpleParser<parsing::JsonHandler> > >(schema);
+ parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonNullFormatter> >, avro::json::JsonNullFormatter> >(schema);
+}
+
+EncoderPtr jsonPrettyEncoder(const ValidSchema& schema)
+{
+ return boost::make_shared<parsing::JsonEncoder<
+ parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonPrettyFormatter> >, avro::json::JsonPrettyFormatter> >(schema);
}
} // namespace avro
http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/test/CodecTests.cc
----------------------------------------------------------------------
diff --git a/lang/c++/test/CodecTests.cc b/lang/c++/test/CodecTests.cc
index 61d29a2..c0ca1e0 100644
--- a/lang/c++/test/CodecTests.cc
+++ b/lang/c++/test/CodecTests.cc
@@ -60,7 +60,7 @@ static const unsigned int count = 10;
/**
* A bunch of tests that share quite a lot of infrastructure between them.
- * The basic idea is to generate avro data for according to a schema and
+ * The basic idea is to generate avro data according to a schema and
* then read back and compare the data with the original. But quite a few
* variations are possible:
* 1. While reading back, one can skip different data elements
@@ -1385,6 +1385,15 @@ struct JsonCodec {
}
};
+struct JsonPrettyCodec {
+ static EncoderPtr newEncoder(const ValidSchema& schema) {
+ return jsonPrettyEncoder(schema);
+ }
+ static DecoderPtr newDecoder(const ValidSchema& schema) {
+ return jsonDecoder(schema);
+ }
+};
+
struct BinaryEncoderResolvingDecoderFactory : public BinaryEncoderFactory {
static DecoderPtr newDecoder(const ValidSchema& schema) {
return resolvingDecoder(schema, schema, binaryDecoder());
@@ -1415,6 +1424,7 @@ void add_tests(boost::unit_test::test_suite& ts)
ADD_TESTS(ts, BinaryCodecFactory, testCodec, data);
ADD_TESTS(ts, ValidatingCodecFactory, testCodec, data);
ADD_TESTS(ts, JsonCodec, testCodec, data);
+ ADD_TESTS(ts, JsonPrettyCodec, testCodec, data);
ADD_TESTS(ts, BinaryEncoderResolvingDecoderFactory, testCodec, data);
ADD_TESTS(ts, ValidatingCodecFactory, testReaderFail, data2);
ADD_TESTS(ts, ValidatingCodecFactory, testWriterFail, data2);
@@ -1500,6 +1510,7 @@ static void testLimitsJsonCodec()
"]}";
ValidSchema schema = parsing::makeValidSchema(s);
testLimits(jsonEncoder(schema), jsonDecoder(schema));
+ testLimits(jsonPrettyEncoder(schema), jsonDecoder(schema));
}
struct JsonData {
@@ -1522,7 +1533,6 @@ static void testJson(const JsonData& data)
{
ValidSchema schema = parsing::makeValidSchema(data.schema);
EncoderPtr e = jsonEncoder(schema);
-
}
} // namespace avro