You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by th...@apache.org on 2012/02/02 20:11:26 UTC
svn commit: r1239768 [1/2] - in /avro/trunk: ./ lang/c++/ lang/c++/api/
lang/c++/impl/ lang/c++/impl/json/ lang/c++/impl/parsing/ lang/c++/test/
Author: thiru
Date: Thu Feb 2 19:11:25 2012
New Revision: 1239768
URL: http://svn.apache.org/viewvc?rev=1239768&view=rev
Log:
AVRO-956. Remove dependency on Flex/Bison
Added:
avro/trunk/lang/c++/impl/json/
avro/trunk/lang/c++/impl/json/JsonDom.cc
avro/trunk/lang/c++/impl/json/JsonDom.hh
avro/trunk/lang/c++/impl/json/JsonIO.cc
avro/trunk/lang/c++/impl/json/JsonIO.hh
avro/trunk/lang/c++/test/JsonTests.cc
avro/trunk/lang/c++/test/SchemaTests.cc
avro/trunk/lang/c++/test/testgentest.cc
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/c++/.gitignore
avro/trunk/lang/c++/CMakeLists.txt
avro/trunk/lang/c++/MainPage.dox
avro/trunk/lang/c++/api/Compiler.hh
avro/trunk/lang/c++/api/CompilerNode.hh
avro/trunk/lang/c++/api/Node.hh
avro/trunk/lang/c++/api/NodeImpl.hh
avro/trunk/lang/c++/api/Stream.hh
avro/trunk/lang/c++/api/SymbolMap.hh
avro/trunk/lang/c++/api/ValidSchema.hh
avro/trunk/lang/c++/impl/Compiler.cc
avro/trunk/lang/c++/impl/CompilerNode.cc
avro/trunk/lang/c++/impl/FileStream.cc
avro/trunk/lang/c++/impl/Node.cc
avro/trunk/lang/c++/impl/ValidSchema.cc
avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
avro/trunk/lang/c++/test/testgen.cc
avro/trunk/lang/c++/test/unittest.cc
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Feb 2 19:11:25 2012
@@ -44,6 +44,8 @@ Avro 1.6.2 (unreleased)
AVRO-926. Java: Fix tests to pass under JDK 7. (cutting)
+ AVRO-956. Remove dependency on Flex/Bison. (thiru)
+
BUG FIXES
AVRO-962. Java: Fix Maven plugin to support string type override.
Modified: avro/trunk/lang/c++/.gitignore
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/.gitignore?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/.gitignore (original)
+++ avro/trunk/lang/c++/.gitignore Thu Feb 2 19:11:25 2012
@@ -1,15 +1,5 @@
-autom4te.cache
-Makefile.in
-configure
-INSTALL
-aclocal.m4
.svn/
-CMakeCache.txt
-CMakeFiles/
-CPackConfig.cmake
-CPackSourceConfig.cmake
-Makefile
build/
-cmake_install.cmake
+doc/
m4/
test.avro
Modified: avro/trunk/lang/c++/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/CMakeLists.txt?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/CMakeLists.txt (original)
+++ avro/trunk/lang/c++/CMakeLists.txt Thu Feb 2 19:11:25 2012
@@ -37,12 +37,12 @@ if (CMAKE_COMPILER_IS_GNUCXX)
endif ()
find_package (Boost 1.38 REQUIRED
- COMPONENTS regex filesystem system program_options)
+ COMPONENTS filesystem system program_options)
include_directories (api ${CMAKE_CURRENT_BINARY_DIR} ${Boost_INCLUDE_DIRS})
set (SOURCE_FILES
- impl/Compiler.cc impl/CompilerNode.cc impl/Node.cc
+ impl/Compiler.cc impl/Node.cc
impl/NodeImpl.cc impl/Resolver.cc impl/ResolverSchema.cc impl/Schema.cc
impl/Types.cc impl/Validator.cc impl/ValidSchema.cc impl/Zigzag.cc
impl/BinaryEncoder.cc impl/BinaryDecoder.cc
@@ -53,7 +53,9 @@ set (SOURCE_FILES
impl/parsing/ValidatingCodec.cc
impl/parsing/JsonCodec.cc
impl/parsing/ResolvingDecoder.cc
- ${CMAKE_CURRENT_BINARY_DIR}/AvroYacc.cc ${CMAKE_CURRENT_BINARY_DIR}/AvroLex.cc)
+ impl/json/JsonIO.cc
+ impl/json/JsonDom.cc
+ )
add_library (avrocpp SHARED ${SOURCE_FILES})
@@ -69,8 +71,6 @@ target_link_libraries (avrocpp ${Boost_L
add_executable (precompile test/precompile.cc)
-add_dependencies(avrocpp parser lexer)
-
target_link_libraries (precompile avrocpp ${Boost_LIBRARIES})
macro (gencpp file ns)
@@ -86,14 +86,6 @@ endmacro (gencpp)
gencpp (bigrecord testgen)
gencpp (bigrecord2 testgen2)
-add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/AvroYacc.cc
- COMMAND bison --defines=${CMAKE_CURRENT_BINARY_DIR}/AvroYacc.hh -o ${CMAKE_CURRENT_BINARY_DIR}/AvroYacc.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/parser/AvroYacc.yy)
-
-add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/AvroLex.cc
- COMMAND flex -oAvroLex.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/parser/AvroLex.ll)
-
macro (gen file ns)
add_custom_command (OUTPUT ${file}.hh
COMMAND avrogencpp
@@ -113,38 +105,33 @@ gen (recursive rec)
gen (reuse ru)
gen (circulardep cd)
-macro (test name)
- add_executable (${name} test/${name}.cc)
- target_link_libraries (${name} avrocpp ${Boost_LIBRARIES})
-endmacro (test)
-
-test (buffertest)
-test (unittest)
-
-add_executable (CodecTests test/CodecTests.cc)
-target_link_libraries (CodecTests avrocpp ${Boost_LIBRARIES})
-
-add_executable (StreamTests test/StreamTests.cc)
-target_link_libraries (StreamTests avrocpp ${Boost_LIBRARIES})
-
-add_executable (SpecificTests test/SpecificTests.cc)
-target_link_libraries (SpecificTests avrocpp ${Boost_LIBRARIES})
-
add_executable (avrogencpp impl/avrogencpp.cc)
target_link_libraries (avrogencpp avrocpp ${Boost_LIBRARIES})
-add_executable (testgentest test/testgen.cc)
-add_dependencies (testgentest testgen testgen2)
-target_link_libraries (testgentest avrocpp ${Boost_LIBRARIES})
+enable_testing()
+
+macro (unittest name)
+ add_executable (${name} test/${name}.cc)
+ target_link_libraries (${name} avrocpp ${Boost_LIBRARIES})
+ add_test (NAME ${name} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${name})
+endmacro (unittest)
+
+unittest (buffertest)
+unittest (unittest)
+unittest (SchemaTests)
+unittest (CodecTests)
+unittest (StreamTests)
+unittest (SpecificTests)
+unittest (DataFileTests)
+unittest (JsonTests)
+unittest (testgentest)
+unittest (AvrogencppTests)
-add_executable (AvrogencppTests test/AvrogencppTests.cc)
+add_dependencies (testgentest testgen testgen2)
add_dependencies (AvrogencppTests bigrecord_hh bigrecord2_hh
union_array_union_hh union_map_union_hh union_conflict_hh
recursive_hh reuse_hh circulardep_hh)
-target_link_libraries (AvrogencppTests avrocpp ${BOOST_LIBRARIES})
-
-add_executable (DataFileTests test/DataFileTests.cc)
-target_link_libraries (DataFileTests avrocpp ${Boost_LIBRARIES})
include (InstallRequiredSystemLibraries)
@@ -167,4 +154,3 @@ if (NOT CMAKE_BUILD_TYPE)
"Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif (NOT CMAKE_BUILD_TYPE)
-
Modified: avro/trunk/lang/c++/MainPage.dox
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/MainPage.dox?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/MainPage.dox (original)
+++ avro/trunk/lang/c++/MainPage.dox Thu Feb 2 19:11:25 2012
@@ -56,9 +56,8 @@ One should be able to build Avro C++ on
In order to build Avro C++, one needs the following:
<ul>
<li>A C++ compiler and runtime libraries.
- <li>Boost library version 1.38 or later. Please see <a href="http://www.boost.org/">http://www.boost.org</a> or your platform's documentation for details on how to set up Boost for your platform.
+ <li>Boost library version 1.38 or later. Apart from the header-only libraries of Boost, Avro C++ requires filesystem, system and program_options libraries. Please see <a href="http://www.boost.org/">http://www.boost.org</a> or your platform's documentation for details on how to set up Boost for your platform.
<li>CMake build tool version 2.6 or later. Please see <a href="http://www.cmake.org">http://www.cmake.org</a> or your platform's documentation for details on how to set up CMake for your system.
- <li>Flex and Bison. For many Linux systems, Flex and Bison are pre-installed. If your platforms does not have Flex or Bison, please install them.
<li>Python. If not already present, please consult your platform-specific documentation on how to install Python on your system.
</ul>
@@ -68,12 +67,9 @@ For Ubuntu Linux, for example, you can h
\li cmake
\li g++
\li libboost-dev
-\li libboost-regex-dev
\li libboost-filesystem-dev
\li libboost-system-dev
\li libboost-program-options-dev
-\li flex
-\li bison
<h3>Installing Avro C++</h3>
<ol>
Modified: avro/trunk/lang/c++/api/Compiler.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Compiler.hh?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/Compiler.hh (original)
+++ avro/trunk/lang/c++/api/Compiler.hh Thu Feb 2 19:11:25 2012
@@ -19,77 +19,18 @@
#ifndef avro_Compiler_hh__
#define avro_Compiler_hh__
-#include "Boost.hh"
-
-#include <FlexLexer.h>
-#include "Types.hh"
-#include "Node.hh"
-#include "CompilerNode.hh"
+#include <stdint.h>
+#include <istream>
namespace avro {
+class InputStream;
+
/// This class is used to implement an avro spec parser using a flex/bison
/// compiler. In order for the lexer to be reentrant, this class provides a
/// lexer object for each parse. The bison parser also uses this class to
/// build up an avro parse tree as the avro spec is parsed.
-class CompilerContext {
-
-
- public:
-
- CompilerContext(std::istream &is) :
- lexer_(&is)
- {}
-
- /// Called by the lexer whenever it encounters text that is not a symbol it recognizes
- /// (names, fieldnames, values to be converted to integers, etc).
- void setText(const char *text) {
- text_ = text;
- }
-
- void addNamedType();
-
- void startType();
- void stopType();
-
- void addType(avro::Type type);
-
- void setSizeAttribute();
- void setNameAttribute();
- void setSymbolsAttribute();
-
- void setFieldsAttribute();
- void setItemsAttribute();
- void setValuesAttribute();
- void setTypesAttribute();
-
- void textContainsFieldName();
-
- const FlexLexer &lexer() const {
- return lexer_;
- }
- FlexLexer &lexer() {
- return lexer_;
- }
-
- const NodePtr &getRoot() const {
- return root_;
- }
-
- private:
-
- typedef boost::ptr_vector<CompilerNode> Stack;
-
- void add(const NodePtr &node);
-
- yyFlexLexer lexer_;
- std::string text_;
-
- NodePtr root_;
- Stack stack_;
-};
-
class ValidSchema;
/// Given a stream comtaining a JSON schema, compiles the schema to a
@@ -105,6 +46,16 @@ void compileJsonSchema(std::istream &is,
bool compileJsonSchema(std::istream &is, ValidSchema &schema, std::string &error);
+ValidSchema compileJsonSchemaFromStream(InputStream& is);
+
+ValidSchema compileJsonSchemaFromMemory(const uint8_t* input, size_t len);
+
+ValidSchema compileJsonSchemaFromString(const char* input);
+
+ValidSchema compileJsonSchemaFromString(const std::string& input);
+
+ValidSchema compileJsonSchemaFromFile(const char* filename);
+
} // namespace avro
#endif
Modified: avro/trunk/lang/c++/api/CompilerNode.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/CompilerNode.hh?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/CompilerNode.hh (original)
+++ avro/trunk/lang/c++/api/CompilerNode.hh Thu Feb 2 19:11:25 2012
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef avro_CompilerNode_hh__
-#define avro_CompilerNode_hh__
-
-#include "NodeConcepts.hh"
-#include "Node.hh"
-
-namespace avro {
-
-
-/// This is a generic "untyped" node that may store values for all possible
-/// attributes of Avro complex types. This allows a Node to be assembled by
-/// the compiler, before it knows what attributes the Node actually contains.
-/// All the Avro types defined (see NodeImpl) may be copy constructed from a
-/// CompilerNode, at which time the attributes actually required by the Avro
-/// type are copied from the CompilerNode, and all unused attributes are
-/// dropped.
-
-class CompilerNode
-{
-
- public:
-
- enum AttributeType {
- NONE,
- FIELDS,
- VALUES,
- ITEMS,
- TYPES
- };
-
- CompilerNode() :
- type_(AVRO_NUM_TYPES),
- attributeType_(NONE)
- {}
-
- CompilerNode(const CompilerNode &rhs) :
- type_(rhs.type_),
- attributeType_(rhs.attributeType_)
- {}
-
-
- AttributeType attributeType() const {
- return attributeType_;
- }
-
- void setAttributeType(AttributeType attributeType) {
- attributeType_ = attributeType;
- }
-
- Type type() const {
- return type_;
- }
-
- void setType(Type type) {
- type_ = type;
- }
-
- void addNode(const NodePtr &node) {
- switch(attributeType_) {
- case FIELDS:
- fieldsAttribute_.add(node);
- break;
- case VALUES:
- valuesAttribute_.add(node);
- break;
- case ITEMS:
- itemsAttribute_.add(node);
- break;
- case TYPES:
- typesAttribute_.add(node);
- break;
-
- default:
- throw Exception("Can't add node if the attribute type is not set");
- }
- }
-
-
- // attribute used by records, enums, symbols, and fixed:
- concepts::SingleAttribute<std::string> nameAttribute_;
-
- // attribute used by fixed:
- concepts::SingleAttribute<int> sizeAttribute_;
-
- // attributes used by records:
- concepts::MultiAttribute<NodePtr> fieldsAttribute_;
- concepts::MultiAttribute<std::string> fieldsNamesAttribute_;
-
- // attribute used by enums:
- concepts::MultiAttribute<std::string> symbolsAttribute_;
-
- // attribute used by arrays:
- concepts::SingleAttribute<NodePtr> itemsAttribute_;
-
- // attribute used by maps:
- concepts::SingleAttribute<NodePtr> valuesAttribute_;
-
- // attribute used by unions:
- concepts::MultiAttribute<NodePtr> typesAttribute_;
-
- Type type_;
- AttributeType attributeType_;
-
-};
-
-NodePtr nodeFromCompilerNode(CompilerNode &compilerNode);
-
-} // namespace avro
-
-#endif
Modified: avro/trunk/lang/c++/api/Node.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Node.hh?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/Node.hh (original)
+++ avro/trunk/lang/c++/api/Node.hh Thu Feb 2 19:11:25 2012
@@ -111,12 +111,10 @@ class Node : private boost::noncopyable
virtual void printBasicInfo(std::ostream &os) const = 0;
- protected:
-
- friend class ValidSchema;
-
virtual void setLeafToSymbolic(int index, const NodePtr &node) = 0;
+ protected:
+
void checkLock() const {
if(locked()) {
throw Exception("Cannot modify locked schema");
@@ -138,4 +136,13 @@ class Node : private boost::noncopyable
} // namespace avro
+namespace std {
+inline std::ostream& operator<<(std::ostream& os, const avro::Node& n)
+{
+ n.printJson(os, 0);
+ return os;
+}
+}
+
+
#endif
Modified: avro/trunk/lang/c++/api/NodeImpl.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/NodeImpl.hh?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/NodeImpl.hh (original)
+++ avro/trunk/lang/c++/api/NodeImpl.hh Thu Feb 2 19:11:25 2012
@@ -20,6 +20,7 @@
#define avro_NodeImpl_hh__
#include <limits>
+#include <set>
#include <boost/weak_ptr.hpp>
#include "Node.hh"
@@ -62,6 +63,13 @@ class NodeImpl : public Node
sizeAttribute_(size)
{ }
+ void swap(NodeImpl& impl) {
+ std::swap(nameAttribute_, impl.nameAttribute_);
+ std::swap(leafAttributes_, impl.leafAttributes_);
+ std::swap(leafNameAttributes_, impl.leafNameAttributes_);
+ std::swap(sizeAttribute_, impl.sizeAttribute_);
+ std::swap(nameIndex_, impl.nameIndex_);
+ }
bool hasName() const {
return NameConcept::hasAttribute;
}
@@ -182,6 +190,9 @@ class NodeSymbolic : public NodeImplSymb
NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoSize())
{ }
+ NodeSymbolic(const HasName &name, const NodePtr n) :
+ NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoSize()), actualNode_(n)
+ { }
SchemaResolution resolve(const Node &reader) const;
void printJson(std::ostream &os, int depth) const;
@@ -230,6 +241,10 @@ class NodeRecord : public NodeImplRecord
}
}
+ void swap(NodeRecord& r) {
+ NodeImplRecord::swap(r);
+ }
+
SchemaResolution resolve(const Node &reader) const;
void printJson(std::ostream &os, int depth) const;
@@ -342,7 +357,60 @@ class NodeUnion : public NodeImplUnion
void printJson(std::ostream &os, int depth) const;
bool isValid() const {
- return (leafAttributes_.size() >= 1);
+ std::set<std::string> seen;
+ if (leafAttributes_.size() >= 1) {
+ for (size_t i = 0; i < leafAttributes_.size(); ++i) {
+ std::string name;
+ const NodePtr& n = leafAttributes_.get(i);
+ switch (n->type()) {
+ case AVRO_STRING:
+ name = "string";
+ break;
+ case AVRO_BYTES:
+ name = "bytes";
+ break;
+ case AVRO_INT:
+ name = "int";
+ break;
+ case AVRO_LONG:
+ name = "long";
+ break;
+ case AVRO_FLOAT:
+ name = "float";
+ break;
+ case AVRO_DOUBLE:
+ name = "double";
+ break;
+ case AVRO_BOOL:
+ name = "bool";
+ break;
+ case AVRO_NULL:
+ name = "null";
+ break;
+ case AVRO_ARRAY:
+ name = "array";
+ break;
+ case AVRO_MAP:
+ name = "map";
+ break;
+ case AVRO_RECORD:
+ case AVRO_ENUM:
+ case AVRO_UNION:
+ case AVRO_FIXED:
+ case AVRO_SYMBOLIC:
+ name = n->name();
+ break;
+ default:
+ return false;
+ }
+ if (seen.find(name) != seen.end()) {
+ return false;
+ }
+ seen.insert(name);
+ }
+ return true;
+ }
+ return false;
}
};
Modified: avro/trunk/lang/c++/api/Stream.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Stream.hh?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/Stream.hh (original)
+++ avro/trunk/lang/c++/api/Stream.hh Thu Feb 2 19:11:25 2012
@@ -154,6 +154,10 @@ std::auto_ptr<OutputStream> fileOutputSt
std::auto_ptr<InputStream> fileInputStream(const char* filename,
size_t bufferSize = 8 * 1024);
+
+std::auto_ptr<InputStream> istreamInputStream(std::istream& in,
+ size_t bufferSize = 8 * 1024);
+
/** A convenience class for reading from an InputStream */
struct StreamReader {
/**
Modified: avro/trunk/lang/c++/api/SymbolMap.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/SymbolMap.hh?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/SymbolMap.hh (original)
+++ avro/trunk/lang/c++/api/SymbolMap.hh Thu Feb 2 19:11:25 2012
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef avro_SymbolMap_hh__
-#define avro_SymbolMap_hh__
-
-#include <map>
-#include <boost/noncopyable.hpp>
-
-#include "Node.hh"
-#include "Schema.hh"
-
-namespace avro {
-
-/// Avro schemas can include types that were previously defined with names in
-/// the same avro schema. In order to identify new types, they are stored in a
-/// map so that the actual type may be identified by name. This class
-/// implements the symbolic name to node mapping.
-///
-
-class SymbolMap : private boost::noncopyable
-{
-
- public:
-
- SymbolMap()
- {}
-
- bool registerSymbol(const NodePtr &node) {
-
- if(node->type() == AVRO_SYMBOLIC) {
- throw Exception("Node must not be a symbolic name");
- }
- const std::string name = node->name();
- if(name.empty()) {
- throw Exception("Node must have a name to be registered");
- }
- bool added = false;
- MapImpl::iterator lb = map_.lower_bound(name);
-
- if(lb == map_.end() || map_.key_comp()(name, lb->first)) {
- map_.insert(lb, std::make_pair(name, node));
- added = true;
- }
- return added;
- }
-
- bool hasSymbol(const std::string &name) const {
- return map_.find(name) != map_.end();
- }
-
- NodePtr locateSymbol(const std::string &name) const {
- MapImpl::const_iterator iter = map_.find(name);
- return (iter == map_.end()) ? NodePtr() : iter->second;
- }
-
- private:
-
- typedef std::map<std::string, NodePtr> MapImpl;
-
- MapImpl map_;
-};
-
-
-} // namespace avro
-
-#endif
Modified: avro/trunk/lang/c++/api/ValidSchema.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/ValidSchema.hh?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/ValidSchema.hh (original)
+++ avro/trunk/lang/c++/api/ValidSchema.hh Thu Feb 2 19:11:25 2012
@@ -24,7 +24,6 @@
namespace avro {
class Schema;
-class SymbolMap;
/// A ValidSchema is basically a non-mutable Schema that has passed some
/// minumum of sanity checks. Once valididated, any Schema that is part of
@@ -37,12 +36,10 @@ class SymbolMap;
/// parsers/serializers, converted to a json schema, etc.
///
-class ValidSchema
-{
- public:
-
+class ValidSchema {
+public:
+ explicit ValidSchema(const NodePtr &root);
explicit ValidSchema(const Schema &schema);
- ValidSchema(const ValidSchema &schema);
ValidSchema();
void setSchema(const Schema &schema);
@@ -56,16 +53,7 @@ class ValidSchema
void toFlatList(std::ostream &os) const;
protected:
-
- bool validate(const NodePtr &node, SymbolMap &symbolMap);
-
NodePtr root_;
-
- private:
-
- // not implemented, only copy construct allowed
- ValidSchema &operator=(const Schema &rhs);
-
};
} // namespace avro
Modified: avro/trunk/lang/c++/impl/Compiler.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/Compiler.cc?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/Compiler.cc (original)
+++ avro/trunk/lang/c++/impl/Compiler.cc Thu Feb 2 19:11:25 2012
@@ -20,170 +20,278 @@
#include "Types.hh"
#include "Schema.hh"
#include "ValidSchema.hh"
+#include "Stream.hh"
+
+#include "json/JsonDom.hh"
extern void yyparse(void *ctx);
+using std::string;
+using std::map;
+using std::vector;
+
namespace avro {
+typedef map<string, NodePtr> SymbolTable;
+
+using json::Entity;
+
// #define DEBUG_VERBOSE
-void
-compileJsonSchema(std::istream &is, ValidSchema &schema)
+static NodePtr makePrimitive(const std::string& t)
{
- if(!is.good()) {
- throw Exception("Input stream is not good");
+ if (t == "null") {
+ return NodePtr(new NodePrimitive(AVRO_NULL));
+ } else if (t == "boolean") {
+ return NodePtr(new NodePrimitive(AVRO_BOOL));
+ } else if (t == "int") {
+ return NodePtr(new NodePrimitive(AVRO_INT));
+ } else if (t == "long") {
+ return NodePtr(new NodePrimitive(AVRO_LONG));
+ } else if (t == "float") {
+ return NodePtr(new NodePrimitive(AVRO_FLOAT));
+ } else if (t == "double") {
+ return NodePtr(new NodePrimitive(AVRO_DOUBLE));
+ } else if (t == "string") {
+ return NodePtr(new NodePrimitive(AVRO_STRING));
+ } else if (t == "bytes") {
+ return NodePtr(new NodePrimitive(AVRO_BYTES));
+ } else {
+ return NodePtr();
}
+}
- CompilerContext myctx(is);
- yyparse(&myctx);
+static NodePtr makeNode(const json::Entity& e, SymbolTable& st);
- Schema s(myctx.getRoot());
- schema.setSchema(s);
+template <typename T>
+concepts::SingleAttribute<T> asSingleAttribute(const T& t)
+{
+ concepts::SingleAttribute<T> n;
+ n.add(t);
+ return n;
}
-bool
-compileJsonSchema(std::istream &is, ValidSchema &schema, std::string &error)
+static NodePtr makeNode(const std::string& t, SymbolTable& st)
{
- bool success = false;
- if(!is.good()) {
- error = "Input stream is not good";
- return false;
- }
-
- try {
- compileJsonSchema(is, schema);
- success = true;
+ NodePtr result = makePrimitive(t);
+ if (result) {
+ return result;
}
- catch (Exception &e) {
- error = e.what();
+ map<string, NodePtr>::const_iterator it = st.find(t);
+ if (it != st.end()) {
+ return NodePtr(new NodeSymbolic(asSingleAttribute(t), it->second));
}
-
- return success;
+ throw Exception(boost::format("Unknown type: %1%") % t);
}
-void
-CompilerContext::add(const NodePtr &node)
+const map<string, Entity>::const_iterator findField(const Entity& e,
+ const map<string, Entity>& m, const string& fieldName)
{
- if(stack_.empty() ) {
- root_ = node;
+ map<string, Entity>::const_iterator it = m.find(fieldName);
+ if (it == m.end()) {
+ throw Exception(boost::format("Missing Json field \"%1%\": %2%") %
+ fieldName % e.toString());
+ } else {
+ return it;
}
- else {
- stack_.back().addNode(node);
- }
}
-void
-CompilerContext::startType()
-{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Start type definition\n";
-#endif
- stack_.push_back(new CompilerNode());
+template<typename T>
+const T& getField(const Entity& e, const map<string, Entity>& m,
+ const string& fieldName)
+{
+ map<string, Entity>::const_iterator it = findField(e, m, fieldName);
+ if (it->second.type() != json::type_traits<T>::type()) {
+ throw Exception(boost::format(
+ "Json field \"%1%\" is not a %2%: %3%") %
+ fieldName % json::type_traits<T>::name() %
+ it->second.toString());
+ } else {
+ return it->second.value<T>();
+ }
}
-void
-CompilerContext::stopType()
-{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Stop type " << stack_.back().type() << '\n';
-#endif
-
- assert(!stack_.empty());
- NodePtr nodePtr(nodeFromCompilerNode(stack_.back()));
- stack_.pop_back();
- add(nodePtr);
+struct Field {
+ const string& name;
+ const NodePtr value;
+ Field(const string& n, const NodePtr& v) : name(n), value(v) { }
+};
+
+static Field makeField(const Entity& e, SymbolTable& st)
+{
+ const map<string, Entity>& m = e.value<map<string, Entity> >();
+ const string& n = getField<string>(e, m, "name");
+ map<string, Entity>::const_iterator it = findField(e, m, "type");
+ return Field(n, makeNode(it->second, st));
+}
+
+static NodePtr makeRecordNode(const Entity& e,
+ const string& name, const map<string, Entity>& m, SymbolTable& st)
+{
+ const vector<Entity>& v = getField<vector<Entity> >(e, m, "fields");
+ concepts::MultiAttribute<string> fieldNames;
+ concepts::MultiAttribute<NodePtr> fieldValues;
+ for (vector<Entity>::const_iterator it = v.begin(); it != v.end(); ++it) {
+ Field f = makeField(*it, st);
+ fieldNames.add(f.name);
+ fieldValues.add(f.value);
+ }
+ return NodePtr(new NodeRecord(asSingleAttribute(name),
+ fieldValues, fieldNames));
}
-void
-CompilerContext::addType(Type type)
-{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Setting type to " << type << '\n';
-#endif
- stack_.back().setType(type);
+static NodePtr makeEnumNode(const Entity& e,
+ const string& name, const map<string, Entity>& m)
+{
+ const vector<Entity>& v = getField<vector<Entity> >(e, m, "symbols");
+ concepts::MultiAttribute<string> symbols;
+ for (vector<Entity>::const_iterator it = v.begin(); it != v.end(); ++it) {
+ if (it->type() != json::etString) {
+ throw Exception(boost::format("Enum symbol not a string: %1%") %
+ it->toString());
+ }
+ symbols.add(it->value<string>());
+ }
+ return NodePtr(new NodeEnum(asSingleAttribute(name), symbols));
}
-void
-CompilerContext::setSizeAttribute()
+static NodePtr makeFixedNode(const Entity& e,
+ const string& name, const map<string, Entity>& m)
{
- int size = atol(text_.c_str());
-#ifdef DEBUG_VERBOSE
- std::cerr << "Setting size to " << size << '\n';
-#endif
- stack_.back().sizeAttribute_.add(size);
+ int v = getField<int64_t>(e, m, "size");
+ if (v <= 0) {
+ throw Exception(boost::format("Size for fixed is not positive: ") %
+ e.toString());
+ }
+
+ return NodePtr(new NodeFixed(asSingleAttribute(name),
+ asSingleAttribute(v)));
+}
+
+static NodePtr makeArrayNode(const Entity& e, const map<string, Entity>& m,
+ SymbolTable& st)
+{
+ map<string, Entity>::const_iterator it = findField(e, m, "items");
+ return NodePtr(new NodeArray(asSingleAttribute(
+ makeNode(it->second, st))));
+}
+
+static NodePtr makeMapNode(const Entity& e, const map<string, Entity>& m,
+ SymbolTable& st)
+{
+ map<string, Entity>::const_iterator it = findField(e, m, "values");
+
+ return NodePtr(new NodeMap(asSingleAttribute(
+ makeNode(it->second, st))));
+}
+
+static NodePtr makeNode(const Entity& e, const map<string, Entity>& m,
+ SymbolTable& st)
+{
+ const string& type = getField<string>(e, m, "type");
+ if (NodePtr result = makePrimitive(type)) {
+ if (m.size() > 1) {
+ throw Exception(boost::format(
+ "Unknown additional Json fields: %1%")
+ % e.toString());
+ } else {
+ return result;
+ }
+ } else if (type == "record" || type == "error" ||
+ type == "enum" || type == "fixed") {
+ const string& name = getField<string>(e, m, "name");
+ NodePtr result;
+ if (type == "record" || type == "error") {
+ result = NodePtr(new NodeRecord());
+ st[name] = result;
+ NodePtr r = makeRecordNode(e, name, m, st);
+ (boost::dynamic_pointer_cast<NodeRecord>(r))->swap(
+ *boost::dynamic_pointer_cast<NodeRecord>(result));
+ } else {
+ result = (type == "enum") ? makeEnumNode(e, name, m) :
+ makeFixedNode(e, name, m);
+ st[name] = result;
+ }
+ return result;
+ } else if (type == "array") {
+ return makeArrayNode(e, m, st);
+ } else if (type == "map") {
+ return makeMapNode(e, m, st);
+ }
+ throw Exception(boost::format("Unknown type definition: %1%")
+ % e.toString());
}
-void
-CompilerContext::addNamedType()
+static NodePtr makeNode(const Entity& e, const vector<Entity>& m,
+ SymbolTable& st)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Adding named type " << text_ << '\n';
-#endif
- stack_.back().setType(AVRO_SYMBOLIC);
- stack_.back().nameAttribute_.add(text_);
+ concepts::MultiAttribute<NodePtr> mm;
+ for (vector<Entity>::const_iterator it = m.begin(); it != m.end(); ++it) {
+ mm.add(makeNode(*it, st));
+ }
+ return NodePtr(new NodeUnion(mm));
}
-void
-CompilerContext::setNameAttribute()
+static NodePtr makeNode(const json::Entity& e, SymbolTable& st)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Setting name to " << text_ << '\n';
-#endif
- stack_.back().nameAttribute_.add(text_);
+ switch (e.type()) {
+ case json::etString:
+ return makeNode(e.value<string>(), st);
+ case json::etObject:
+ return makeNode(e, e.value<map<string, Entity> >(), st);
+ case json::etArray:
+ return makeNode(e, e.value<vector<Entity> >(), st);
+ default:
+ throw Exception(boost::format("Invalid Avro type: %1%") % e.toString());
+ }
}
-void
-CompilerContext::setSymbolsAttribute()
+ValidSchema compileJsonSchemaFromStream(InputStream& is)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Adding enum symbol " << text_ << '\n';
-#endif
- stack_.back().symbolsAttribute_.add(text_);
+ json::Entity e = json::loadEntity(is);
+ SymbolTable st;
+ NodePtr n = makeNode(e, st);
+ return ValidSchema(n);
}
-void
-CompilerContext::setValuesAttribute()
+ValidSchema compileJsonSchemaFromMemory(const uint8_t* input, size_t len)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Ready for map type\n";
-#endif
- stack_.back().setAttributeType(CompilerNode::VALUES);
+ return compileJsonSchemaFromStream(*memoryInputStream(input, len));
}
-void
-CompilerContext::setTypesAttribute()
+ValidSchema compileJsonSchemaFromString(const char* input)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Ready for union types\n";
-#endif
- stack_.back().setAttributeType(CompilerNode::TYPES);
+ return compileJsonSchemaFromMemory(reinterpret_cast<const uint8_t*>(input),
+ ::strlen(input));
}
-void
-CompilerContext::setItemsAttribute()
+static ValidSchema compile(std::istream& is)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Ready for array type\n";
-#endif
- stack_.back().setAttributeType(CompilerNode::ITEMS);
+ std::auto_ptr<InputStream> in = istreamInputStream(is);
+ return compileJsonSchemaFromStream(*in);
}
-void
-CompilerContext::setFieldsAttribute()
+void
+compileJsonSchema(std::istream &is, ValidSchema &schema)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Ready for record fields\n";
-#endif
- stack_.back().setAttributeType(CompilerNode::FIELDS);
+ if (!is.good()) {
+ throw Exception("Input stream is not good");
+ }
+
+ schema = compile(is);
}
-void
-CompilerContext::textContainsFieldName()
+bool
+compileJsonSchema(std::istream &is, ValidSchema &schema, std::string &error)
{
-#ifdef DEBUG_VERBOSE
- std::cerr << "Setting field name to " << text_ << '\n';
-#endif
- stack_.back().fieldsNamesAttribute_.add(text_);
+ try {
+ compileJsonSchema(is, schema);
+ return true;
+ } catch (const Exception &e) {
+ error = e.what();
+ return false;
+ }
+
}
} // namespace avro
Modified: avro/trunk/lang/c++/impl/CompilerNode.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/CompilerNode.cc?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/CompilerNode.cc (original)
+++ avro/trunk/lang/c++/impl/CompilerNode.cc Thu Feb 2 19:11:25 2012
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-#include "CompilerNode.hh"
-#include "NodeImpl.hh"
-
-namespace avro {
-
-NodePtr
-nodeFromCompilerNode(CompilerNode &node)
-{
- NodePtr ptr;
-
- switch(node.type()) {
-
- case AVRO_ARRAY:
- ptr.reset ( new NodeArray(node.itemsAttribute_));
- break;
-
- case AVRO_ENUM:
- ptr.reset ( new NodeEnum(node.nameAttribute_, node.symbolsAttribute_));
- break;
-
- case AVRO_FIXED:
- ptr.reset ( new NodeFixed(node.nameAttribute_, node.sizeAttribute_));
- break;
-
- case AVRO_MAP:
- ptr.reset ( new NodeMap(node.valuesAttribute_));
- break;
-
- case AVRO_RECORD:
- ptr.reset ( new NodeRecord(node.nameAttribute_, node.fieldsAttribute_, node.fieldsNamesAttribute_));
- break;
-
- case AVRO_UNION:
- ptr.reset ( new NodeUnion(node.typesAttribute_));
- break;
-
- case AVRO_SYMBOLIC:
- ptr.reset ( new NodeSymbolic(node.nameAttribute_));
- break;
-
- default:
- if(isPrimitive(node.type())) {
- ptr.reset ( new NodePrimitive(node.type()));
- }
- else {
- throw Exception("Unknown type in nodeFromCompilerNode");
- }
- break;
- }
-
- return ptr;
-}
-
-} // namespace avro
Modified: avro/trunk/lang/c++/impl/FileStream.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/FileStream.cc?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/FileStream.cc (original)
+++ avro/trunk/lang/c++/impl/FileStream.cc Thu Feb 2 19:11:25 2012
@@ -25,12 +25,81 @@
#define O_BINARY 0
#endif
+using std::auto_ptr;
+using std::istream;
+
namespace avro {
-class FileInputStream : public InputStream {
+namespace stream {
+struct BufferCopyIn {
+ virtual ~BufferCopyIn() { }
+ virtual void seek(size_t len) = 0;
+ virtual bool read(uint8_t* b, size_t toRead, size_t& actual) = 0;
+
+};
+
+struct FileBufferCopyIn : public BufferCopyIn {
+ const int fd_;
+
+ FileBufferCopyIn(const char* filename) :
+ fd_(open(filename, O_RDONLY | O_BINARY)) {
+ if (fd_ < 0) {
+ throw Exception(boost::format("Cannot open file: %1%") %
+ ::strerror(errno));
+ }
+ }
+
+ ~FileBufferCopyIn() {
+ ::close(fd_);
+ }
+
+ void seek(size_t len) {
+ off_t r = ::lseek(fd_, len, SEEK_CUR);
+ if (r == static_cast<off_t>(-1)) {
+ throw Exception(boost::format("Cannot skip file: %1%") %
+ strerror(errno));
+ }
+ }
+
+ bool read(uint8_t* b, size_t toRead, size_t& actual) {
+ int n = ::read(fd_, b, toRead);
+ if (n > 0) {
+ actual = n;
+ return true;
+ }
+ return false;
+ }
+
+};
+
+struct IStreamBufferCopyIn : public BufferCopyIn {
+ istream& is_;
+
+ IStreamBufferCopyIn(istream& is) : is_(is) {
+ }
+
+ void seek(size_t len) {
+ if (! is_.seekg(len, std::ios_base::cur)) {
+ throw Exception("Cannot skip stream");
+ }
+ }
+
+ bool read(uint8_t* b, size_t toRead, size_t& actual) {
+ is_.read(reinterpret_cast<char*>(b), toRead);
+ if (is_.bad()) {
+ return false;
+ }
+ actual = is_.gcount();
+ return (! is_.eof() || actual != 0);
+ }
+
+};
+
+}
+
+class BufferCopyInInputStream : public InputStream {
const size_t bufferSize_;
uint8_t* const buffer_;
- /// Input file descriptor
- int in_;
+ auto_ptr<stream::BufferCopyIn> in_;
size_t byteCount_;
uint8_t* next_;
size_t available_;
@@ -56,11 +125,7 @@ class FileInputStream : public InputStre
void skip(size_t len) {
while (len > 0) {
if (available_ == 0) {
- off_t r = ::lseek(in_, len, SEEK_CUR);
- if (r == static_cast<off_t>(-1)) {
- throw Exception(boost::format("Cannot skip file: %1%") %
- strerror(errno));
- }
+ in_->seek(len);
byteCount_ += len;
return;
}
@@ -75,8 +140,8 @@ class FileInputStream : public InputStre
size_t byteCount() const { return byteCount_; }
bool fill() {
- int n = ::read(in_, buffer_, bufferSize_);
- if (n > 0) {
+ size_t n = 0;
+ if (in_->read(buffer_, bufferSize_, n)) {
next_ = buffer_;
available_ = n;
return true;
@@ -86,21 +151,15 @@ class FileInputStream : public InputStre
public:
- FileInputStream(const char* filename, size_t bufferSize) :
+ BufferCopyInInputStream(auto_ptr<stream::BufferCopyIn>& in, size_t bufferSize) :
bufferSize_(bufferSize),
buffer_(new uint8_t[bufferSize]),
- in_(open(filename, O_RDONLY | O_BINARY)),
+ in_(in),
byteCount_(0),
next_(buffer_),
- available_(0) {
- if (in_ < 0) {
- throw Exception(boost::format("Cannot open file: %1%") %
- ::strerror(errno));
- }
- }
+ available_(0) { }
- ~FileInputStream() {
- ::close(in_);
+ ~BufferCopyInInputStream() {
delete[] buffer_;
}
};
@@ -161,17 +220,24 @@ public:
}
};
-std::auto_ptr<InputStream> fileInputStream(const char* filename,
+auto_ptr<InputStream> fileInputStream(const char* filename,
+ size_t bufferSize)
+{
+ auto_ptr<stream::BufferCopyIn> in(new stream::FileBufferCopyIn(filename));
+ return auto_ptr<InputStream>( new BufferCopyInInputStream(in, bufferSize));
+}
+
+auto_ptr<InputStream> istreamInputStream(istream& is,
size_t bufferSize)
{
- return std::auto_ptr<InputStream>(
- new FileInputStream(filename, bufferSize));
+ auto_ptr<stream::BufferCopyIn> in(new stream::IStreamBufferCopyIn(is));
+ return auto_ptr<InputStream>( new BufferCopyInInputStream(in, bufferSize));
}
-std::auto_ptr<OutputStream> fileOutputStream(const char* filename,
+auto_ptr<OutputStream> fileOutputStream(const char* filename,
size_t bufferSize)
{
- return std::auto_ptr<OutputStream>(
+ return auto_ptr<OutputStream>(
new FileOutputStream(filename, bufferSize));
}
Modified: avro/trunk/lang/c++/impl/Node.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/Node.cc?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/Node.cc (original)
+++ avro/trunk/lang/c++/impl/Node.cc Thu Feb 2 19:11:25 2012
@@ -16,21 +16,26 @@
* limitations under the License.
*/
-#include <boost/regex.hpp>
#include "Node.hh"
namespace avro {
+using std::string;
+
Node::~Node()
{ }
void
Node::checkName(const std::string &name) const
{
- const boost::regex exp("[A-Za-z_][A-Za-z0-9_]*");
- if(!name.empty() && !boost::regex_match(name, exp)) {
- throw Exception("Names must match [A-Za-z_][A-Za-z0-9_]*");
+ string::const_iterator it = name.begin();
+ if (it != name.end() && (isalpha(*it) || *it == '_')) {
+ while ((++it != name.end()) && (isalnum(*it) || *it == '_'));
+ if (it == name.end()) {
+ return;
+ }
}
+ throw Exception("Names must match [A-Za-z_][A-Za-z0-9_]*");
}
} // namespace avro
Modified: avro/trunk/lang/c++/impl/ValidSchema.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/ValidSchema.cc?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/ValidSchema.cc (original)
+++ avro/trunk/lang/c++/impl/ValidSchema.cc Thu Feb 2 19:11:25 2012
@@ -19,69 +19,57 @@
#include <boost/format.hpp>
#include "ValidSchema.hh"
-#include "SymbolMap.hh"
#include "Schema.hh"
#include "Node.hh"
-namespace avro {
-
-ValidSchema::ValidSchema(const Schema &schema) :
- root_(schema.root())
-{
- SymbolMap symbolMap;
- validate(root_, symbolMap);
-}
+using std::string;
+using std::make_pair;
+using boost::format;
+using boost::shared_ptr;
+using boost::static_pointer_cast;
-ValidSchema::ValidSchema(const ValidSchema &schema) :
- root_(schema.root())
-{ }
+namespace avro {
-ValidSchema::ValidSchema() :
- root_(NullSchema().root())
-{ }
+typedef std::map<std::string, NodePtr> SymbolMap;
-void
-ValidSchema::setSchema(const Schema &schema)
+static bool validate(const NodePtr &node, SymbolMap &symbolMap)
{
- const NodePtr &node(schema.root());
- SymbolMap symbolMap;
- validate(schema.root(), symbolMap);
- root_ = node;
-}
-
-bool
-ValidSchema::validate(const NodePtr &node, SymbolMap &symbolMap)
-{
- if(!node) {
- root_.reset(new NodePrimitive(AVRO_NULL));
+ if (! node->isValid()) {
+ throw Exception(format("Schema is invalid, due to bad node of type %1%")
+ % node->type());
}
- if(!node->isValid()) {
- throw Exception( boost::format("Schema is invalid, due to bad node of type %1%") % node->type());
- }
- if(node->hasName()) {
- if(node->type() == AVRO_SYMBOLIC) {
- if(!symbolMap.hasSymbol(node->name())) {
- throw Exception( boost::format("Symbolic name \"%1%\" is unknown") % node->name());
+ if (node->hasName()) {
+ const string& nm = node->name();
+ SymbolMap::iterator it = symbolMap.find(nm);
+ bool found = it != symbolMap.end() && nm == it->first;
+
+ if (node->type() == AVRO_SYMBOLIC) {
+ if (! found) {
+ throw Exception(format("Symbolic name \"%1%\" is unknown") %
+ node->name());
}
- boost::shared_ptr<NodeSymbolic> symNode = boost::static_pointer_cast<NodeSymbolic>(node);
+ shared_ptr<NodeSymbolic> symNode =
+ static_pointer_cast<NodeSymbolic>(node);
// if the symbolic link is already resolved, we return true,
// otherwise returning false will force it to be resolved
return symNode->isSet();
}
- bool registered = symbolMap.registerSymbol(node);
- if(!registered) {
+
+ if (found) {
return false;
}
+ symbolMap.insert(it, make_pair(nm, node));
}
+
node->lock();
size_t leaves = node->leaves();
- for(size_t i = 0; i < leaves; ++i) {
+ for (size_t i = 0; i < leaves; ++i) {
const NodePtr &leaf(node->leafAt(i));
- if(! validate(leaf, symbolMap)) {
+ if (! validate(leaf, symbolMap)) {
// if validate returns false it means a node with this name already
// existed in the map, instead of keeping this node twice in the
@@ -89,14 +77,41 @@ ValidSchema::validate(const NodePtr &nod
// links that could not be easily freed), replace this node with a
// symbolic link to the original one.
- NodePtr redirect = symbolMap.locateSymbol(leaf->name());
- node->setLeafToSymbolic(i, redirect);
+ node->setLeafToSymbolic(i, symbolMap.find(leaf->name())->second);
}
}
return true;
}
+static void validate(const NodePtr& p)
+{
+ SymbolMap m;
+ validate(p, m);
+}
+
+ValidSchema::ValidSchema(const NodePtr &root) : root_(root)
+{
+ validate(root_);
+}
+
+ValidSchema::ValidSchema(const Schema &schema) : root_(schema.root())
+{
+ validate(root_);
+}
+
+ValidSchema::ValidSchema() : root_(NullSchema().root())
+{
+ validate(root_);
+}
+
+void
+ValidSchema::setSchema(const Schema &schema)
+{
+ root_ = schema.root();
+ validate(root_);
+}
+
void
ValidSchema::toJson(std::ostream &os) const
{
Added: avro/trunk/lang/c++/impl/json/JsonDom.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonDom.cc?rev=1239768&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonDom.cc (added)
+++ avro/trunk/lang/c++/impl/json/JsonDom.cc Thu Feb 2 19:11:25 2012
@@ -0,0 +1,150 @@
+#include "JsonDom.hh"
+
+#include <stdexcept>
+
+#include <string.h>
+
+#include "Stream.hh"
+#include "JsonIO.hh"
+
+namespace avro {
+namespace json {
+
+Entity readEntity(JsonParser& p)
+{
+ switch (p.peek()) {
+ case JsonParser::tkNull:
+ p.advance();
+ return Entity();
+ case JsonParser::tkBool:
+ p.advance();
+ return Entity(p.boolValue());
+ case JsonParser::tkLong:
+ p.advance();
+ return Entity(p.longValue());
+ case JsonParser::tkDouble:
+ p.advance();
+ return Entity(p.doubleValue());
+ case JsonParser::tkString:
+ p.advance();
+ return Entity(p.stringValue());
+ case JsonParser::tkArrayStart:
+ {
+ p.advance();
+ std::vector<Entity> v;
+ while (p.peek() != JsonParser::tkArrayEnd) {
+ v.push_back(readEntity(p));
+ }
+ p.advance();
+ return Entity(v);
+ }
+ case JsonParser::tkObjectStart:
+ {
+ p.advance();
+ std::map<std::string, Entity> v;
+ while (p.peek() != JsonParser::tkObjectEnd) {
+ p.advance();
+ std::string k = p.stringValue();
+ Entity n = readEntity(p);
+ v.insert(std::make_pair(k, n));
+ }
+ p.advance();
+ return Entity(v);
+ }
+ default:
+ throw std::domain_error(JsonParser::toString(p.peek()));
+ }
+
+}
+
+Entity loadEntity(const char* text)
+{
+ return loadEntity(reinterpret_cast<const uint8_t*>(text), ::strlen(text));
+}
+
+Entity loadEntity(InputStream& in)
+{
+ JsonParser p;
+ p.init(in);
+ return readEntity(p);
+}
+
+Entity loadEntity(const uint8_t* text, size_t len)
+{
+ std::auto_ptr<InputStream> in = memoryInputStream(text, len);
+ return loadEntity(*in);
+}
+
+void writeEntity(JsonGenerator& g, const Entity& n)
+{
+ switch (n.type()) {
+ case etNull:
+ g.encodeNull();
+ break;
+ case etBool:
+ g.encodeBool(n.value<bool>());
+ break;
+ case etLong:
+ g.encodeNumber(n.value<int64_t>());
+ break;
+ case etDouble:
+ g.encodeNumber(n.value<double>());
+ break;
+ case etString:
+ g.encodeString(n.value<std::string>());
+ break;
+ case etArray:
+ {
+ g.arrayStart();
+ const std::vector<Entity>& v = n.value<std::vector<Entity> >();
+ for (std::vector<Entity>::const_iterator it = v.begin();
+ it != v.end(); ++it) {
+ writeEntity(g, *it);
+ }
+ g.arrayEnd();
+ }
+ break;
+ case etObject:
+ {
+ g.objectStart();
+ const std::map<std::string, Entity>& v =
+ n.value<std::map<std::string, Entity> >();
+ for (std::map<std::string, Entity>::const_iterator it = v.begin();
+ it != v.end(); ++it) {
+ g.encodeString(it->first);
+ writeEntity(g, it->second);
+ }
+ g.objectEnd();
+ }
+ break;
+ }
+}
+
+std::string Entity::toString() const
+{
+ std::auto_ptr<OutputStream> out = memoryOutputStream();
+ JsonGenerator g;
+ g.init(*out);
+ writeEntity(g, *this);
+ g.flush();
+ std::auto_ptr<InputStream> in = memoryInputStream(*out);
+ const uint8_t *p = 0;
+ size_t n = 0;
+ size_t c = 0;
+ while (in->next(&p, &n)) {
+ c += n;
+ }
+ std::string result;
+ result.resize(c);
+ c = 0;
+ std::auto_ptr<InputStream> in2 = memoryInputStream(*out);
+ while (in2->next(&p, &n)) {
+ ::memcpy(&result[c], p, n);
+ c += n;
+ }
+ return result;
+}
+
+}
+}
+
Added: avro/trunk/lang/c++/impl/json/JsonDom.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonDom.hh?rev=1239768&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonDom.hh (added)
+++ avro/trunk/lang/c++/impl/json/JsonDom.hh Thu Feb 2 19:11:25 2012
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef avro_json_JsonDom_hh__
+#define avro_json_JsonDom_hh__
+
+#include <stdint.h>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "boost/any.hpp"
+
+namespace avro {
+
+class InputStream;
+
+namespace json {
+
+class JsonParser;
+class JsonGenerator;
+
+enum EntityType {
+ etNull,
+ etBool,
+ etLong,
+ etDouble,
+ etString,
+ etArray,
+ etObject
+};
+
+class Entity {
+ EntityType type_;
+ boost::any value_;
+public:
+ Entity() : type_(etNull) { }
+ Entity(bool v) : type_(etBool), value_(v) { }
+ Entity(int64_t v) : type_(etLong), value_(v) { }
+ Entity(double v) : type_(etDouble), value_(v) { }
+ Entity(const std::string& v) : type_(etString), value_(v) { }
+ Entity(const std::vector<Entity> v) : type_(etArray), value_(v) { }
+ Entity(const std::map<std::string, Entity> v) : type_(etObject), value_(v) {
+ }
+
+
+ EntityType type() const { return type_; }
+
+ template <typename T>
+ const T& value() const {
+ return *boost::any_cast<T>(&value_);
+ }
+
+ template <typename T>
+ T& value() {
+ return *boost::any_cast<T>(&value_);
+ }
+
+ template <typename T>
+ void set(const T& v) {
+ *boost::any_cast<T>(&value_) = v;
+ }
+
+ std::string toString() const;
+};
+
+template <typename T>
+struct type_traits {
+};
+
+template <> struct type_traits<std::string> {
+ static EntityType type() { return etString; }
+ static const char* name() { return "string"; }
+};
+
+template <> struct type_traits<std::vector<Entity> > {
+ static EntityType type() { return etArray; }
+ static const char* name() { return "array"; }
+};
+
+template <> struct type_traits<int64_t> {
+ static EntityType type() { return etLong; }
+ static const char* name() { return "integer"; }
+};
+
+Entity readEntity(JsonParser& p);
+
+Entity loadEntity(InputStream& in);
+Entity loadEntity(const char* text);
+Entity loadEntity(const uint8_t* text, size_t len);
+
+void writeEntity(JsonGenerator& g, const Entity& n);
+
+}
+}
+
+#endif
+
+
Added: avro/trunk/lang/c++/impl/json/JsonIO.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonIO.cc?rev=1239768&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonIO.cc (added)
+++ avro/trunk/lang/c++/impl/json/JsonIO.cc Thu Feb 2 19:11:25 2012
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "JsonIO.hh"
+
+namespace avro {
+namespace json {
+
+const char* const
+JsonParser::tokenNames[] = {
+ "Null",
+ "Bool",
+ "Integer",
+ "Double",
+ "String",
+ "Array start",
+ "Array end",
+ "Object start",
+ "Object end",
+};
+
+char JsonParser::next()
+{
+ char ch = hasNext ? nextChar : ' ';
+ while (isspace(ch)) {
+ ch = in_.read();
+ }
+ hasNext = false;
+ return ch;
+}
+
+JsonParser::Token JsonParser::doAdvance()
+{
+ char ch = next();
+ if (ch == ']') {
+ if (curState == stArray0 || stArrayN) {
+ curState = stateStack.top();
+ stateStack.pop();
+ return tkArrayEnd;
+ } else {
+ unexpected(ch);
+ }
+ } else if (ch == '}') {
+ if (curState == stObject0 || stObjectN) {
+ curState = stateStack.top();
+ stateStack.pop();
+ return tkObjectEnd;
+ } else {
+ unexpected(ch);
+ }
+ } else if (ch == ',') {
+ if (curState != stObjectN && curState != stArrayN) {
+ unexpected(ch);
+ }
+ if (curState == stObjectN) {
+ curState = stObject0;
+ }
+ ch = next();
+ } else if (ch == ':') {
+ if (curState != stKey) {
+ unexpected(ch);
+ }
+ curState = stObjectN;
+ ch = next();
+ }
+
+ if (curState == stObject0) {
+ if (ch != '"') {
+ unexpected(ch);
+ }
+ curState = stKey;
+ } else if (curState == stArray0) {
+ curState = stArrayN;
+ }
+
+ switch (ch) {
+ case '[':
+ stateStack.push(curState);
+ curState = stArray0;
+ return tkArrayStart;
+ case '{':
+ stateStack.push(curState);
+ curState = stObject0;
+ return tkObjectStart;
+ case '"':
+ return tryString();
+ case 't':
+ bv = true;
+ return tryLiteral("rue", 3, tkBool);
+ case 'f':
+ bv = false;
+ return tryLiteral("alse", 4, tkBool);
+ case 'n':
+ return tryLiteral("ull", 3, tkNull);
+ default:
+ if (isdigit(ch) || '-') {
+ return tryNumber(ch);
+ } else {
+ unexpected(ch);
+ }
+ }
+}
+
+JsonParser::Token JsonParser::tryNumber(char ch)
+{
+ sv.clear();
+ sv.push_back(ch);
+
+ hasNext = false;
+ int state = (ch == '-') ? 0 : (ch == 0) ? 1 : 2;
+ for (; ;) {
+ switch (state) {
+ case 0:
+ if (in_.hasMore()) {
+ ch = in_.read();
+ if (isdigit(ch)) {
+ state = (ch == 0) ? 1 : 2;
+ sv.push_back(ch);
+ continue;
+ }
+ hasNext = true;
+ }
+ break;
+ case 1:
+ if (in_.hasMore()) {
+ ch = in_.read();
+ if (ch == '.') {
+ state = 3;
+ sv.push_back(ch);
+ continue;
+ }
+ hasNext = true;
+ }
+ break;
+ case 2:
+ if (in_.hasMore()) {
+ ch = in_.read();
+ if (isdigit(ch)) {
+ sv.push_back(ch);
+ continue;
+ } else if (ch == '.') {
+ state = 3;
+ sv.push_back(ch);
+ continue;
+ }
+ hasNext = true;
+ }
+ break;
+ case 3:
+ case 6:
+ if (in_.hasMore()) {
+ ch = in_.read();
+ if (isdigit(ch)) {
+ sv.push_back(ch);
+ state++;
+ continue;
+ }
+ hasNext = true;
+ }
+ break;
+ case 4:
+ if (in_.hasMore()) {
+ ch = in_.read();
+ if (isdigit(ch)) {
+ sv.push_back(ch);
+ continue;
+ } else if (ch == 'e' || ch == 'E') {
+ sv.push_back(ch);
+ state = 5;
+ continue;
+ }
+ hasNext = true;
+ }
+ break;
+ case 5:
+ if (in_.hasMore()) {
+ ch = in_.read();
+ if (ch == '+' || ch == '-') {
+ sv.push_back(ch);
+ state = 6;
+ continue;
+ } else if (isdigit(ch)) {
+ sv.push_back(ch);
+ state = 7;
+ continue;
+ }
+ hasNext = true;
+ }
+ break;
+ case 7:
+ if (in_.hasMore()) {
+ ch = in_.read();
+ if (isdigit(ch)) {
+ sv.push_back(ch);
+ continue;
+ }
+ hasNext = true;
+ }
+ break;
+ }
+ if (state == 1 || state == 2 || state == 4 || state == 7) {
+ if (hasNext) {
+ nextChar = ch;
+ }
+ std::istringstream iss(sv);
+ if (state == 1 || state == 2) {
+ iss >> lv;
+ return tkLong;
+ } else {
+ iss >> dv;
+ return tkDouble;
+ }
+ } else {
+ if (hasNext) {
+ unexpected(ch);
+ } else {
+ throw Exception("Unexpected EOF");
+ }
+ }
+ }
+}
+
+JsonParser::Token JsonParser::tryString()
+{
+ sv.clear();
+ for ( ; ;) {
+ char ch = in_.read();
+ if (ch == '"') {
+ return tkString;
+ } else if (ch == '\\') {
+ ch = in_.read();
+ switch (ch) {
+ case '"':
+ case '\\':
+ case '/':
+ sv.push_back(ch);
+ continue;
+ case 'b':
+ sv.push_back('\b');
+ continue;
+ case 'f':
+ sv.push_back('\f');
+ continue;
+ case 'n':
+ sv.push_back('\n');
+ continue;
+ case 'r':
+ sv.push_back('\r');
+ continue;
+ case 't':
+ sv.push_back('\t');
+ continue;
+ case 'U':
+ {
+ unsigned int n = 0;
+ char e[4];
+ in_.readBytes(reinterpret_cast<uint8_t*>(e), 4);
+ for (int i = 0; i < 4; i++) {
+ n *= 16;
+ char c = e[i];
+ if (isdigit(c)) {
+ n += c - '0';
+ } else if (c >= 'a' && c <= 'f') {
+ n += c - 'a' + 10;
+ } else if (c >= 'A' && c <= 'F') {
+ n += c - 'A' + 10;
+ } else {
+ unexpected(c);
+ }
+ }
+ sv.push_back(n);
+ }
+ break;
+ default:
+ unexpected(ch);
+ }
+ } else {
+ sv.push_back(ch);
+ }
+ }
+}
+
+void JsonParser::unexpected(unsigned char c)
+{
+ std::ostringstream oss;
+ oss << "Unexpected character in json " << toHex(c / 16) << toHex(c % 16);
+ throw Exception(oss.str());
+}
+
+JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n, Token tk)
+{
+ char c[100];
+ in_.readBytes(reinterpret_cast<uint8_t*>(c), n);
+ for (size_t i = 0; i < n; ++i) {
+ if (c[i] != exp[i]) {
+ unexpected(c[i]);
+ }
+ }
+ if (in_.hasMore()) {
+ nextChar = in_.read();
+ if (isdigit(nextChar) || isalpha(nextChar)) {
+ unexpected(nextChar);
+ }
+ hasNext = true;
+ }
+ return tk;
+}
+
+}
+}
+
Added: avro/trunk/lang/c++/impl/json/JsonIO.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonIO.hh?rev=1239768&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonIO.hh (added)
+++ avro/trunk/lang/c++/impl/json/JsonIO.hh Thu Feb 2 19:11:25 2012
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef avro_json_JsonIO_hh__
+#define avro_json_JsonIO_hh__
+
+#include <stack>
+#include <string>
+#include <sstream>
+#include <boost/utility.hpp>
+
+#include "Stream.hh"
+
+namespace avro {
+namespace json {
+
+inline char toHex(unsigned int n) {
+ return (n < 10) ? (n + '0') : (n + 'a' - 10);
+}
+
+
+class JsonParser : boost::noncopyable {
+public:
+ enum Token {
+ tkNull,
+ tkBool,
+ tkLong,
+ tkDouble,
+ tkString,
+ tkArrayStart,
+ tkArrayEnd,
+ tkObjectStart,
+ tkObjectEnd
+ };
+
+private:
+ enum State {
+ stValue, // Expect a data type
+ stArray0, // Expect a data type or ']'
+ stArrayN, // Expect a ',' or ']'
+ stObject0, // Expect a string or a '}'
+ stObjectN, // Expect a ',' or '}'
+ stKey // Expect a ':'
+ };
+ std::stack<State> stateStack;
+ State curState;
+ bool hasNext;
+ char nextChar;
+ bool peeked;
+
+ StreamReader in_;
+ Token curToken;
+ bool bv;
+ int64_t lv;
+ double dv;
+ std::string sv;
+
+ Token doAdvance();
+ Token tryLiteral(const char exp[], size_t n, Token tk);
+ Token tryNumber(char ch);
+ Token tryString();
+ void unexpected(unsigned char ch);
+ char next();
+
+public:
+ JsonParser() : curState(stValue), hasNext(false), peeked(false) { }
+
+ void init(InputStream& is) {
+ in_.reset(is);
+ }
+
+ Token advance() {
+ if (! peeked) {
+ curToken = doAdvance();
+ } else {
+ peeked = false;
+ }
+ return curToken;
+ }
+
+ Token peek() {
+ if (! peeked) {
+ curToken = doAdvance();
+ peeked = true;
+ }
+ return curToken;
+ }
+
+ bool boolValue() {
+ return bv;
+ }
+
+ Token cur() {
+ return curToken;
+ }
+
+ double doubleValue() {
+ return dv;
+ }
+
+ int64_t longValue() {
+ return lv;
+ }
+
+ std::string stringValue() {
+ return sv;
+ }
+
+ static const char* const tokenNames[];
+
+ static const char* toString(Token tk) {
+ return tokenNames[tk];
+ }
+};
+
+class JsonGenerator {
+ StreamWriter out_;
+ enum State {
+ stStart,
+ stArray0,
+ stArrayN,
+ stMap0,
+ stMapN,
+ stKey,
+ };
+
+ std::stack<State> stateStack;
+ State top;
+
+ void write(const char *b, const char* p) {
+ if (b != p) {
+ out_.writeBytes(reinterpret_cast<const uint8_t*>(b), p - b);
+ }
+ }
+
+ void escape(char c, const char* b, const char *p) {
+ write(b, p);
+ out_.write('\\');
+ out_.write(c);
+ }
+
+ void escapeCtl(char c) {
+ out_.write('\\');
+ out_.write('U');
+ out_.write('0');
+ out_.write('0');
+ out_.write(toHex((static_cast<unsigned char>(c)) / 16));
+ out_.write(toHex((static_cast<unsigned char>(c)) % 16));
+ }
+
+ void doEncodeString(const std::string& s) {
+ const char* b = &s[0];
+ const char* e = b + s.size();
+ out_.write('"');
+ for (const char* p = b; p != e; p++) {
+ switch (*p) {
+ case '\\':
+ case '"':
+ case '/':
+ escape(*p, b, p);
+ break;
+ case '\b':
+ escape('b', b, p);
+ break;
+ case '\f':
+ escape('f', b, p);
+ break;
+ case '\n':
+ escape('n', b, p);
+ break;
+ case '\r':
+ escape('r', b, p);
+ break;
+ case '\t':
+ escape('t', b, p);
+ break;
+ default:
+ if (! iscntrl(*p)) {
+ continue;
+ }
+ write(b, p);
+ escapeCtl(*p);
+ break;
+ }
+ b = p + 1;
+ }
+ write(b, e);
+ out_.write('"');
+ }
+
+ void sep() {
+ if (top == stArrayN) {
+ out_.write(',');
+ } else if (top == stArray0) {
+ top = stArrayN;
+ }
+ }
+
+ void sep2() {
+ if (top == stKey) {
+ top = stMapN;
+ }
+ }
+
+public:
+ JsonGenerator() : top(stStart) { }
+
+ void init(OutputStream& os) {
+ out_.reset(os);
+ }
+
+ void flush() {
+ out_.flush();
+ }
+
+ void encodeNull() {
+ sep();
+ out_.writeBytes(reinterpret_cast<const uint8_t*>("null"), 4);
+ sep2();
+ }
+
+ void encodeBool(bool b) {
+ sep();
+ if (b) {
+ out_.writeBytes(reinterpret_cast<const uint8_t*>("true"), 4);
+ } else {
+ out_.writeBytes(reinterpret_cast<const uint8_t*>("false"), 5);
+ }
+ sep2();
+ }
+
+ template <typename T>
+ void encodeNumber(T t) {
+ sep();
+ std::ostringstream oss;
+ oss << t;
+ const std::string& s = oss.str();
+ out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size());
+ sep2();
+ }
+
+ void encodeString(const std::string& s) {
+ if (top == stMap0) {
+ top = stKey;
+ } else if (top == stMapN) {
+ out_.write(',');
+ top = stKey;
+ } else if (top == stKey) {
+ top = stMapN;
+ } else {
+ sep();
+ }
+ doEncodeString(s);
+ if (top == stKey) {
+ out_.write(':');
+ }
+ }
+
+ void encodeBinary(const uint8_t* bytes, size_t len) {
+ sep();
+ out_.write('"');
+ const uint8_t* e = bytes + len;
+ while (bytes != e) {
+ escapeCtl(*bytes++);
+ }
+ out_.write('"');
+ sep2();
+ }
+
+ void arrayStart() {
+ sep();
+ stateStack.push(top);
+ top = stArray0;
+ out_.write('[');
+ }
+
+ void arrayEnd() {
+ top = stateStack.top();
+ stateStack.pop();
+ out_.write(']');
+ sep2();
+ }
+
+ void objectStart() {
+ sep();
+ stateStack.push(top);
+ top = stMap0;
+ out_.write('{');
+ }
+
+ void objectEnd() {
+ top = stateStack.top();
+ stateStack.pop();
+ out_.write('}');
+ sep2();
+ }
+
+};
+
+}
+}
+
+#endif
Modified: avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/parsing/JsonCodec.cc?rev=1239768&r1=1239767&r2=1239768&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/parsing/JsonCodec.cc (original)
+++ avro/trunk/lang/c++/impl/parsing/JsonCodec.cc Thu Feb 2 19:11:25 2012
@@ -19,7 +19,6 @@
#define __STDC_LIMIT_MACROS
#include <string>
-#include <stack>
#include <map>
#include <algorithm>
#include <ctype.h>
@@ -27,7 +26,6 @@
#include <boost/make_shared.hpp>
#include <boost/weak_ptr.hpp>
#include <boost/any.hpp>
-#include <boost/utility.hpp>
#include "ValidatingCodec.hh"
#include "Symbol.hh"
@@ -36,6 +34,8 @@
#include "Encoder.hh"
#include "NodeImpl.hh"
+#include "../json/JsonIO.hh"
+
namespace avro {
using boost::make_shared;
@@ -51,7 +51,9 @@ using std::string;
using std::reverse;
using std::ostringstream;
using std::istringstream;
-using std::stack;
+
+using avro::json::JsonParser;
+using avro::json::JsonGenerator;
class JsonGrammarGenerator : public ValidatingGrammarGenerator {
Production doGenerate(const NodePtr& n,
@@ -158,406 +160,6 @@ Production JsonGrammarGenerator::doGener
}
}
-static char toHex(unsigned int n) {
- return (n < 10) ? (n + '0') : (n + 'a' - 10);
-}
-
-class JsonParser : boost::noncopyable {
-public:
- enum Token {
- tkNull,
- tkBool,
- tkLong,
- tkDouble,
- tkString,
- tkArrayStart,
- tkArrayEnd,
- tkObjectStart,
- tkObjectEnd
- };
-
-private:
- enum State {
- stValue, // Expect a data type
- stArray0, // Expect a data type or ']'
- stArrayN, // Expect a ',' or ']'
- stObject0, // Expect a string or a '}'
- stObjectN, // Expect a ',' or '}'
- stKey // Expect a ':'
- };
- stack<State> stateStack;
- State curState;
- bool hasNext;
- char nextChar;
- bool peeked;
-
- StreamReader in_;
- Token curToken;
- bool bv;
- int64_t lv;
- double dv;
- string sv;
-
- Token doAdvance();
- Token tryLiteral(const char exp[], size_t n, Token tk);
- Token tryNumber(char ch);
- Token tryString();
- void unexpected(unsigned char ch);
- char next();
-
-public:
- JsonParser() : curState(stValue), hasNext(false), peeked(false) { }
-
- void init(InputStream& is) {
- in_.reset(is);
- }
-
- Token advance() {
- if (! peeked) {
- curToken = doAdvance();
- } else {
- peeked = false;
- }
- return curToken;
- }
-
- Token peek() {
- if (! peeked) {
- curToken = doAdvance();
- peeked = true;
- }
- return curToken;
- }
-
- bool boolValue() {
- return bv;
- }
-
- Token cur() {
- return curToken;
- }
-
- double doubleValue() {
- return dv;
- }
-
- int64_t longValue() {
- return lv;
- }
-
- string stringValue() {
- return sv;
- }
-
- static const char* const tokenNames[];
-
- static const char* toString(Token tk) {
- return tokenNames[tk];
- }
-};
-
-const char* const
-JsonParser::tokenNames[] = {
- "Null",
- "Bool",
- "Integer",
- "Double",
- "String",
- "Array start",
- "Array end",
- "Object start",
- "Object end",
-};
-
-char JsonParser::next()
-{
- char ch = hasNext ? nextChar : ' ';
- while (isspace(ch)) {
- ch = in_.read();
- }
- hasNext = false;
- return ch;
-}
-
-JsonParser::Token JsonParser::doAdvance()
-{
- char ch = next();
- if (ch == ']') {
- if (curState == stArray0 || stArrayN) {
- curState = stateStack.top();
- stateStack.pop();
- return tkArrayEnd;
- } else {
- unexpected(ch);
- }
- } else if (ch == '}') {
- if (curState == stObject0 || stObjectN) {
- curState = stateStack.top();
- stateStack.pop();
- return tkObjectEnd;
- } else {
- unexpected(ch);
- }
- } else if (ch == ',') {
- if (curState != stObjectN && curState != stArrayN) {
- unexpected(ch);
- }
- if (curState == stObjectN) {
- curState = stObject0;
- }
- ch = next();
- } else if (ch == ':') {
- if (curState != stKey) {
- unexpected(ch);
- }
- curState = stObjectN;
- ch = next();
- }
-
- if (curState == stObject0) {
- if (ch != '"') {
- unexpected(ch);
- }
- curState = stKey;
- } else if (curState == stArray0) {
- curState = stArrayN;
- }
-
- switch (ch) {
- case '[':
- stateStack.push(curState);
- curState = stArray0;
- return tkArrayStart;
- case '{':
- stateStack.push(curState);
- curState = stObject0;
- return tkObjectStart;
- case '"':
- return tryString();
- case 't':
- bv = true;
- return tryLiteral("rue", 3, tkBool);
- case 'f':
- bv = false;
- return tryLiteral("alse", 4, tkBool);
- case 'n':
- return tryLiteral("ull", 3, tkNull);
- default:
- if (isdigit(ch) || '-') {
- return tryNumber(ch);
- } else {
- unexpected(ch);
- }
- }
-}
-
-JsonParser::Token JsonParser::tryNumber(char ch)
-{
- sv.clear();
- sv.push_back(ch);
-
- hasNext = false;
- int state = (ch == '-') ? 0 : (ch == 0) ? 1 : 2;
- for (; ;) {
- switch (state) {
- case 0:
- if (in_.hasMore()) {
- ch = in_.read();
- if (isdigit(ch)) {
- state = (ch == 0) ? 1 : 2;
- sv.push_back(ch);
- continue;
- }
- hasNext = true;
- }
- break;
- case 1:
- if (in_.hasMore()) {
- ch = in_.read();
- if (ch == '.') {
- state = 3;
- sv.push_back(ch);
- continue;
- }
- hasNext = true;
- }
- break;
- case 2:
- if (in_.hasMore()) {
- ch = in_.read();
- if (isdigit(ch)) {
- sv.push_back(ch);
- continue;
- } else if (ch == '.') {
- state = 3;
- sv.push_back(ch);
- continue;
- }
- hasNext = true;
- }
- break;
- case 3:
- case 6:
- if (in_.hasMore()) {
- ch = in_.read();
- if (isdigit(ch)) {
- sv.push_back(ch);
- state++;
- continue;
- }
- hasNext = true;
- }
- break;
- case 4:
- if (in_.hasMore()) {
- ch = in_.read();
- if (isdigit(ch)) {
- sv.push_back(ch);
- continue;
- } else if (ch == 'e' || ch == 'E') {
- sv.push_back(ch);
- state = 5;
- continue;
- }
- hasNext = true;
- }
- break;
- case 5:
- if (in_.hasMore()) {
- in_.read();
- ch = next();
- if (ch == '+' || ch == '-') {
- sv.push_back(ch);
- state = 6;
- continue;
- } else if (isdigit(ch)) {
- sv.push_back(ch);
- state = 7;
- continue;
- }
- hasNext = true;
- }
- break;
- case 7:
- if (in_.hasMore()) {
- in_.read();
- ch = next();
- if (isdigit(ch)) {
- sv.push_back(ch);
- continue;
- }
- hasNext = true;
- }
- break;
- }
- if (state == 1 || state == 2 || state == 4 || state == 7) {
- if (hasNext) {
- nextChar = ch;
- }
- istringstream iss(sv);
- if (state == 1 || state == 2) {
- iss >> lv;
- return tkLong;
- } else {
- iss >> dv;
- return tkDouble;
- }
- } else {
- if (hasNext) {
- unexpected(ch);
- } else {
- throw Exception("Unexpected EOF");
- }
- }
- }
-}
-
-JsonParser::Token JsonParser::tryString()
-{
- sv.clear();
- for ( ; ;) {
- char ch = in_.read();
- if (ch == '"') {
- return tkString;
- } else if (ch == '\\') {
- ch = in_.read();
- switch (ch) {
- case '"':
- case '\\':
- case '/':
- sv.push_back(ch);
- continue;
- case 'b':
- sv.push_back('\b');
- continue;
- case 'f':
- sv.push_back('\f');
- continue;
- case 'n':
- sv.push_back('\n');
- continue;
- case 'r':
- sv.push_back('\r');
- continue;
- case 't':
- sv.push_back('\t');
- continue;
- case 'U':
- {
- unsigned int n = 0;
- char e[4];
- in_.readBytes(reinterpret_cast<uint8_t*>(e), 4);
- for (int i = 0; i < 4; i++) {
- n *= 16;
- char c = e[i];
- if (isdigit(c)) {
- n += c - '0';
- } else if (c >= 'a' && c <= 'f') {
- n += c - 'a' + 10;
- } else if (c >= 'A' && c <= 'F') {
- n += c - 'A' + 10;
- } else {
- unexpected(c);
- }
- }
- sv.push_back(n);
- }
- break;
- default:
- unexpected(ch);
- }
- } else {
- sv.push_back(ch);
- }
- }
-}
-
-void JsonParser::unexpected(unsigned char c)
-{
- ostringstream oss;
- oss << "Unexpected character in json " << toHex(c / 16) << toHex(c % 16);
- throw Exception(oss.str());
-}
-
-JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n, Token tk)
-{
- char c[100];
- in_.readBytes(reinterpret_cast<uint8_t*>(c), n);
- for (size_t i = 0; i < n; ++i) {
- if (c[i] != exp[i]) {
- unexpected(c[i]);
- }
- }
- if (in_.hasMore()) {
- nextChar = in_.read();
- if (isdigit(nextChar) || isalpha(nextChar)) {
- unexpected(nextChar);
- }
- hasNext = true;
- }
- return tk;
-}
-
static void expectToken(JsonParser& in, JsonParser::Token tk)
{
if (in.advance() != tk) {
@@ -874,189 +476,6 @@ size_t JsonDecoder<P>::decodeUnionIndex(
return result;
}
-class JsonGenerator {
- StreamWriter out_;
- enum State {
- stStart,
- stArray0,
- stArrayN,
- stMap0,
- stMapN,
- stKey,
- };
-
- stack<State> stateStack;
- State top;
-
- void write(const char *b, const char* p) {
- if (b != p) {
- out_.writeBytes(reinterpret_cast<const uint8_t*>(b), p - b);
- }
- }
-
- void escape(char c, const char* b, const char *p) {
- write(b, p);
- out_.write('\\');
- out_.write(c);
- }
-
- void escapeCtl(char c) {
- out_.write('\\');
- out_.write('U');
- out_.write('0');
- out_.write('0');
- out_.write(toHex((static_cast<unsigned char>(c)) / 16));
- out_.write(toHex((static_cast<unsigned char>(c)) % 16));
- }
-
- void doEncodeString(const std::string& s) {
- const char* b = &s[0];
- const char* e = b + s.size();
- out_.write('"');
- for (const char* p = b; p != e; p++) {
- switch (*p) {
- case '\\':
- case '"':
- case '/':
- escape(*p, b, p);
- break;
- case '\b':
- escape('b', b, p);
- break;
- case '\f':
- escape('f', b, p);
- break;
- case '\n':
- escape('n', b, p);
- break;
- case '\r':
- escape('r', b, p);
- break;
- case '\t':
- escape('t', b, p);
- break;
- default:
- if (! iscntrl(*p)) {
- continue;
- }
- write(b, p);
- escapeCtl(*p);
- break;
- }
- b = p + 1;
- }
- write(b, e);
- out_.write('"');
- }
-
- void sep() {
- if (top == stArrayN) {
- out_.write(',');
- } else if (top == stArray0) {
- top = stArrayN;
- }
- }
-
- void sep2() {
- if (top == stKey) {
- top = stMapN;
- }
- }
-
-public:
- JsonGenerator() : top(stStart) { }
-
- void init(OutputStream& os) {
- out_.reset(os);
- }
-
- void flush() {
- out_.flush();
- }
-
- void encodeNull() {
- sep();
- out_.writeBytes(reinterpret_cast<const uint8_t*>("null"), 4);
- sep2();
- }
-
- void encodeBool(bool b) {
- sep();
- if (b) {
- out_.writeBytes(reinterpret_cast<const uint8_t*>("true"), 4);
- } else {
- out_.writeBytes(reinterpret_cast<const uint8_t*>("false"), 5);
- }
- sep2();
- }
-
- template <typename T>
- void encodeNumber(T t) {
- sep();
- ostringstream oss;
- oss << t;
- const std::string& s = oss.str();
- out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size());
- sep2();
- }
-
- void encodeString(const std::string& s) {
- if (top == stMap0) {
- top = stKey;
- } else if (top == stMapN) {
- out_.write(',');
- top = stKey;
- } else if (top == stKey) {
- top = stMapN;
- } else {
- sep();
- }
- doEncodeString(s);
- if (top == stKey) {
- out_.write(':');
- }
- }
-
- void encodeBinary(const uint8_t* bytes, size_t len) {
- sep();
- out_.write('"');
- const uint8_t* e = bytes + len;
- while (bytes != e) {
- escapeCtl(*bytes++);
- }
- out_.write('"');
- sep2();
- }
-
- void arrayStart() {
- sep();
- stateStack.push(top);
- top = stArray0;
- out_.write('[');
- }
-
- void arrayEnd() {
- top = stateStack.top();
- stateStack.pop();
- out_.write(']');
- sep2();
- }
-
- void objectStart() {
- sep();
- stateStack.push(top);
- top = stMap0;
- out_.write('{');
- }
-
- void objectEnd() {
- top = stateStack.top();
- stateStack.pop();
- out_.write('}');
- sep2();
- }
-
-};
class JsonHandler {
JsonGenerator& generator_;