You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by th...@apache.org on 2013/06/13 21:02:46 UTC
svn commit: r1492821 - in /avro/trunk: CHANGES.txt
lang/c++/impl/json/JsonIO.cc lang/c++/impl/json/JsonIO.hh
lang/c++/impl/parsing/JsonCodec.cc lang/c++/test/CodecTests.cc
Author: thiru
Date: Thu Jun 13 19:02:46 2013
New Revision: 1492821
URL: http://svn.apache.org/r1492821
Log:
AVRO-1290. Handling NaN and positive and negative infinities in C++ Json
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/c++/impl/json/JsonIO.cc
avro/trunk/lang/c++/impl/json/JsonIO.hh
avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
avro/trunk/lang/c++/test/CodecTests.cc
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Jun 13 19:02:46 2013
@@ -62,6 +62,8 @@ Trunk (not yet released)
AVRO-1324. C: Handle namespaces in schema parsing.
(Ben Walsh via dcreager)
+ AVRO-1290. Handling NaN and positive and negative infinities in C++ Json (Daniel Russel via thiru)
+
BUG FIXES
AVRO-1296. Python: Fix schemas retrieved from protocol types
Modified: avro/trunk/lang/c++/impl/json/JsonIO.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonIO.cc?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonIO.cc (original)
+++ avro/trunk/lang/c++/impl/json/JsonIO.cc Thu Jun 13 19:02:46 2013
@@ -21,6 +21,8 @@
namespace avro {
namespace json {
+using std::ostringstream;
+
const char* const
JsonParser::tokenNames[] = {
"Null",
@@ -44,6 +46,25 @@ char JsonParser::next()
return ch;
}
+void JsonParser::expectToken(Token tk)
+{
+ if (advance() != tk) {
+ if (tk == tkDouble && cur() == tkString
+ && (sv == "Infinity" || sv == "-Infinity" || sv == "NaN")) {
+ curToken = tkDouble;
+ dv = sv == "Infinity" ? std::numeric_limits<double>::infinity() :
+ sv == "-Infinity" ? -std::numeric_limits<double>::infinity() :
+ std::numeric_limits<double>::quiet_NaN();
+ return;
+ }
+ ostringstream oss;
+ oss << "Incorrect token in the stream. Expected: "
+ << JsonParser::toString(tk) << ", found "
+ << JsonParser::toString(cur());
+ throw Exception(oss.str());
+ }
+}
+
JsonParser::Token JsonParser::doAdvance()
{
char ch = next();
@@ -122,14 +143,14 @@ JsonParser::Token JsonParser::tryNumber(
sv.push_back(ch);
hasNext = false;
- int state = (ch == '-') ? 0 : (ch == 0) ? 1 : 2;
+ int state = (ch == '-') ? 0 : (ch == '0') ? 1 : 2;
for (; ;) {
switch (state) {
case 0:
if (in_.hasMore()) {
ch = in_.read();
if (isdigit(ch)) {
- state = (ch == 0) ? 1 : 2;
+ state = (ch == '0') ? 1 : 2;
sv.push_back(ch);
continue;
}
Modified: avro/trunk/lang/c++/impl/json/JsonIO.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonIO.hh?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonIO.hh (original)
+++ avro/trunk/lang/c++/impl/json/JsonIO.hh Thu Jun 13 19:02:46 2013
@@ -102,6 +102,8 @@ public:
return curToken;
}
+ void expectToken(Token tk);
+
bool boolValue() {
return bv;
}
Modified: avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/parsing/JsonCodec.cc?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/parsing/JsonCodec.cc (original)
+++ avro/trunk/lang/c++/impl/parsing/JsonCodec.cc Thu Jun 13 19:02:46 2013
@@ -26,6 +26,7 @@
#include <boost/make_shared.hpp>
#include <boost/weak_ptr.hpp>
#include <boost/any.hpp>
+#include <boost/math/special_functions/fpclassify.hpp>
#include "ValidatingCodec.hh"
#include "Symbol.hh"
@@ -160,13 +161,7 @@ Production JsonGrammarGenerator::doGener
static void expectToken(JsonParser& in, JsonParser::Token tk)
{
- if (in.advance() != tk) {
- ostringstream oss;
- oss << "Incorrect token in the stream. Expected: "
- << JsonParser::toString(tk) << ", found "
- << JsonParser::toString(in.cur());
- throw Exception(oss.str());
- }
+ in.expectToken(tk);
}
class JsonDecoderHandler {
@@ -573,14 +568,30 @@ template<typename P>
void JsonEncoder<P>::encodeFloat(float f)
{
parser_.advance(Symbol::sFloat);
- out_.encodeNumber(f);
+ if (f == std::numeric_limits<float>::infinity()) {
+ out_.encodeString("Infinity");
+ } else if (f == -std::numeric_limits<float>::infinity()) {
+ out_.encodeString("-Infinity");
+ } else if (boost::math::isnan(f)) {
+ out_.encodeString("NaN");
+ } else {
+ out_.encodeNumber(f);
+ }
}
template<typename P>
void JsonEncoder<P>::encodeDouble(double d)
{
parser_.advance(Symbol::sDouble);
- out_.encodeNumber(d);
+ if (d == std::numeric_limits<double>::infinity()) {
+ out_.encodeString("Infinity");
+ } else if (d == -std::numeric_limits<double>::infinity()) {
+ out_.encodeString("-Infinity");
+ } else if (boost::math::isnan(d)) {
+ out_.encodeString("NaN");
+ } else {
+ out_.encodeNumber(d);
+ }
}
template<typename P>
Modified: avro/trunk/lang/c++/test/CodecTests.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/test/CodecTests.cc?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/test/CodecTests.cc (original)
+++ avro/trunk/lang/c++/test/CodecTests.cc Thu Jun 13 19:02:46 2013
@@ -36,8 +36,24 @@
#include <boost/test/unit_test.hpp>
#include <boost/test/parameterized_test.hpp>
#include <boost/random/mersenne_twister.hpp>
+#include <boost/math/special_functions/fpclassify.hpp>
namespace avro {
+
+/*
+void dump(const OutputStream& os)
+{
+ std::auto_ptr<InputStream> in = memoryInputStream(os);
+ const char *b;
+ size_t n;
+ std::cout << os.byteCount() << std::endl;
+ while (in->next(reinterpret_cast<const uint8_t**>(&b), &n)) {
+ std::cout << std::string(b, n);
+ }
+ std::cout << std::endl;
+}
+*/
+
namespace parsing {
static const unsigned int count = 10;
@@ -589,20 +605,6 @@ struct TestData4 {
unsigned int depth;
};
-/*
-static void dump(const OutputStream& os)
-{
- std::auto_ptr<InputStream> in = memoryInputStream(os);
- const char *b;
- size_t n;
- std::cout << os.byteCount() << std::endl;
- while (in->next(reinterpret_cast<const uint8_t**>(&b), &n)) {
- std::cout << std::string(b, n);
- }
- std::cout << std::endl;
-}
-*/
-
template<typename CodecFactory>
void testCodec(const TestData& td) {
static int testNo = 0;
@@ -1418,6 +1420,57 @@ static void testStreamLifetimes()
}
+static void testLimits(const EncoderPtr& e, const DecoderPtr& d)
+{
+ std::auto_ptr<OutputStream> s1 = memoryOutputStream();
+ {
+ e->init(*s1);
+ e->encodeDouble(std::numeric_limits<double>::infinity());
+ e->encodeDouble(-std::numeric_limits<double>::infinity());
+ e->encodeDouble(std::numeric_limits<double>::quiet_NaN());
+ e->encodeFloat(std::numeric_limits<float>::infinity());
+ e->encodeFloat(-std::numeric_limits<float>::infinity());
+ e->encodeFloat(std::numeric_limits<float>::quiet_NaN());
+ e->flush();
+ }
+
+ {
+ std::auto_ptr<InputStream> s2 = memoryInputStream(*s1);
+ d->init(*s2);
+ BOOST_CHECK_EQUAL(d->decodeDouble(),
+ std::numeric_limits<double>::infinity());
+ BOOST_CHECK_EQUAL(d->decodeDouble(),
+ -std::numeric_limits<double>::infinity());
+ BOOST_CHECK(boost::math::isnan(d->decodeDouble()));
+ BOOST_CHECK_EQUAL(d->decodeFloat(),
+ std::numeric_limits<float>::infinity());
+ BOOST_CHECK_EQUAL(d->decodeFloat(),
+ -std::numeric_limits<float>::infinity());
+ BOOST_CHECK(boost::math::isnan(d->decodeFloat()));
+ }
+
+}
+
+static void testLimitsBinaryCodec()
+{
+ testLimits(binaryEncoder(), binaryDecoder());
+}
+
+static void testLimitsJsonCodec()
+{
+ const char* s = "{ \"type\": \"record\", \"name\": \"r\", \"fields\": ["
+ "{ \"name\": \"d1\", \"type\": \"double\" },"
+ "{ \"name\": \"d2\", \"type\": \"double\" },"
+ "{ \"name\": \"d3\", \"type\": \"double\" },"
+ "{ \"name\": \"f1\", \"type\": \"float\" },"
+ "{ \"name\": \"f2\", \"type\": \"float\" },"
+ "{ \"name\": \"f3\", \"type\": \"float\" }"
+ "]}";
+ ValidSchema schema = parsing::makeValidSchema(s);
+ testLimits(jsonEncoder(schema), jsonDecoder(schema));
+}
+
+
} // namespace avro
boost::unit_test::test_suite*
@@ -1428,6 +1481,8 @@ init_unit_test_suite( int argc, char* ar
test_suite* ts= BOOST_TEST_SUITE("Avro C++ unit tests for codecs");
avro::parsing::add_tests(*ts);
ts->add(BOOST_TEST_CASE(avro::testStreamLifetimes));
+ ts->add(BOOST_TEST_CASE(avro::testLimitsBinaryCodec));
+ ts->add(BOOST_TEST_CASE(avro::testLimitsJsonCodec));
return ts;
}