You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2011/12/09 21:41:14 UTC
svn commit: r1212616 - in /avro/trunk: ./ doc/src/content/xdocs/
lang/java/avro/src/main/java/org/apache/avro/
lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/
lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/ lang/jav...
Author: cutting
Date: Fri Dec 9 20:41:13 2011
New Revision: 1212616
URL: http://svn.apache.org/viewvc?rev=1212616&view=rev
Log:
AVRO-965. Java: Enhance IDL to support properties for protocols and messages. Contributed by George Fletcher.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/doc/src/content/xdocs/idl.xml
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj
avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl
avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr
avro/trunk/share/test/schemas/simple.avpr
avro/trunk/share/test/schemas/social.avdl
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Dec 9 20:41:13 2011
@@ -19,6 +19,9 @@ Avro 1.6.2 (unreleased)
types, we use the strings "NaN", "Infinity" and "-Infinity" in
schemas. These are also permitted in IDL. (cutting)
+ AVRO-965. Java: Enhance IDL to support properties for protocols
+ and messages. (George Fletcher via cutting)
+
BUG FIXES
AVRO-962. Java: Fix Maven plugin to support string type override.
Modified: avro/trunk/doc/src/content/xdocs/idl.xml
URL: http://svn.apache.org/viewvc/avro/trunk/doc/src/content/xdocs/idl.xml?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/doc/src/content/xdocs/idl.xml (original)
+++ avro/trunk/doc/src/content/xdocs/idl.xml Fri Dec 9 20:41:13 2011
@@ -52,11 +52,11 @@
Avro Protocol file with extension <code>.avpr</code>.
</p>
<p>
- To convert a <code>.avdl</code> file into a <code>.avpr</code> file, it must be processed by the
+ To convert a <code>.avdl</code> file into a <code>.avpr</code> file, it may be processed by the
<code>idl</code> tool. For example:
</p>
<source>
-$ java -jar avroj-tools-1.4.0.jar idl src/test/idl/input/namespaces.avdl /tmp/namespaces.avpr
+$ java -jar avroj-tools.jar idl src/test/idl/input/namespaces.avdl /tmp/namespaces.avpr
$ head /tmp/namespaces.avpr
{
"protocol" : "TestNamespace",
@@ -66,6 +66,25 @@ $ head /tmp/namespaces.avpr
The <code>idl</code> tool can also process input to and from <em>stdin</em> and <em>stdout</em>.
See <code>idl --help</code> for full usage information.
</p>
+ <p>A Maven plugin is also provided to compile .avdl files. To
+ use it, add something like the following to your pom.xml:</p>
+<source><![CDATA[
+<build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>idl-protocol</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+</build>
+]]></source>
</section>
</section> <!-- end overview -->
@@ -358,6 +377,9 @@ record MyRecord {
string @aliases(["oldField", "ancientField"]) myNewField;
}
</source>
+ <p>Some annotations like those listed above are handled
+ specially. All other annotations are added as properties to
+ the protocol, message, schema or field.</p>
</section>
</section>
<section id="example">
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java Fri Dec 9 20:41:13 2011
@@ -28,6 +28,9 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.HashSet;
import org.apache.avro.Schema.Field;
import org.codehaus.jackson.JsonNode;
@@ -58,17 +61,30 @@ public class Protocol {
/** The version of the protocol specification implemented here. */
public static final long VERSION = 1;
+ // Support properties for both Protocol and Message objects
+ private static final Set<String> MESSAGE_RESERVED = new HashSet<String>();
+ static {
+ Collections.addAll(MESSAGE_RESERVED,
+ "doc", "response","request", "errors", "one-way");
+ }
+
/** A protocol message. */
public class Message {
private String name;
private String doc;
private Schema request;
-
+ private final Schema.Props props = new Schema.Props(MESSAGE_RESERVED);
+
/** Construct a message. */
- private Message(String name, String doc, Schema request) {
+ private Message(String name, String doc,
+ Map<String,String> propMap, Schema request) {
this.name = name;
this.doc = doc;
this.request = request;
+
+ if (propMap != null) // copy props
+ for (Map.Entry<String,String> prop : propMap.entrySet())
+ this.addProp(prop.getKey(), prop.getValue());
}
/** The name of this message. */
@@ -85,6 +101,17 @@ public class Protocol {
/** Returns true if this is a one-way message, with no response or errors.*/
public boolean isOneWay() { return true; }
+ /** Return the value of the named property in this field or null. */
+ public synchronized String getProp(String name) { return props.get(name); }
+ /** Add a property with the given name to this field. */
+ public synchronized void addProp(String name, String value) {
+ props.add(name, value);
+ }
+ /** Return the defined properties as an unmodifieable Map. */
+ public Map<String,String> getProps() {
+ return Collections.unmodifiableMap(props);
+ }
+
public String toString() {
try {
StringWriter writer = new StringWriter();
@@ -98,9 +125,8 @@ public class Protocol {
}
void toJson(JsonGenerator gen) throws IOException {
gen.writeStartObject();
-
if (doc != null) gen.writeStringField("doc", doc);
-
+ props.write(gen); // write out properties
gen.writeFieldName("request");
request.fieldsToJson(types, gen);
@@ -118,11 +144,12 @@ public class Protocol {
if (!(o instanceof Message)) return false;
Message that = (Message)o;
return this.name.equals(that.name)
- && this.request.equals(that.request);
+ && this.request.equals(that.request)
+ && props.equals(that.props);
}
public int hashCode() {
- return name.hashCode() + request.hashCode();
+ return name.hashCode() + request.hashCode() + props.hashCode();
}
public String getDoc() { return doc; }
@@ -134,9 +161,9 @@ public class Protocol {
private Schema errors;
/** Construct a message. */
- private TwoWayMessage(String name, String doc, Schema request,
- Schema response, Schema errors) {
- super(name, doc, request);
+ private TwoWayMessage(String name, String doc, Map<String,String> propMap,
+ Schema request, Schema response, Schema errors) {
+ super(name, doc, propMap, request);
this.response = response;
this.errors = errors;
}
@@ -190,6 +217,14 @@ public class Protocol {
SYSTEM_ERRORS = Schema.createUnion(errors);
}
+ private static final Set<String> PROTOCOL_RESERVED = new HashSet<String>();
+ static {
+ Collections.addAll(PROTOCOL_RESERVED,
+ "namespace", "protocol", "doc",
+ "messages","types", "errors");
+ }
+ Schema.Props props = new Schema.Props(PROTOCOL_RESERVED);
+
private Protocol() {}
public Protocol(String name, String doc, String namespace) {
@@ -227,14 +262,55 @@ public class Protocol {
public Map<String,Message> getMessages() { return messages; }
/** Create a one-way message. */
+ @Deprecated
public Message createMessage(String name, String doc, Schema request) {
- return new Message(name, doc, request);
+ return createMessage(name, doc, new LinkedHashMap<String,String>(),request);
+ }
+ /** Create a one-way message. */
+ public Message createMessage(String name, String doc,
+ Map<String,String> propMap, Schema request) {
+ return new Message(name, doc, propMap, request);
}
/** Create a two-way message. */
+ @Deprecated
public Message createMessage(String name, String doc, Schema request,
Schema response, Schema errors) {
- return new TwoWayMessage(name, doc, request, response, errors);
+ return createMessage(name, doc, new LinkedHashMap<String,String>(),
+ request, response, errors);
+ }
+ /** Create a two-way message. */
+ public Message createMessage(String name, String doc,
+ Map<String,String> propMap, Schema request,
+ Schema response, Schema errors) {
+ return new TwoWayMessage(name, doc, propMap, request, response, errors);
+ }
+
+ /**
+ * Returns the value of the named property in this schema.
+ * Returns <tt>null</tt> if there is no property with that name.
+ * @param name
+ */
+ public synchronized String getProp(String name) {
+ return props.get(name);
+ }
+
+ /**
+ * Adds a property with the given name <tt>name</tt> and
+ * value <tt>value</tt>. Neither <tt>name</tt> nor <tt>value</tt> can be
+ * <tt>null</tt>. It is illegal to add a property if another with
+ * the same name but different value already exists in this schema.
+ *
+ * @param name The name of the property to add
+ * @param value The value for the property to add
+ */
+ public synchronized void addProp(String name, String value) {
+ props.add(name, value);
+ }
+
+ /** Return the defined properties as an unmodifieable Map. */
+ public Map<String,String> getProps() {
+ return Collections.unmodifiableMap(props);
}
public boolean equals(Object o) {
@@ -244,12 +320,13 @@ public class Protocol {
return this.name.equals(that.name)
&& this.namespace.equals(that.namespace)
&& this.types.equals(that.types)
- && this.messages.equals(that.messages);
+ && this.messages.equals(that.messages)
+ && this.props.equals(this.props);
}
public int hashCode() {
return name.hashCode() + namespace.hashCode()
- + types.hashCode() + messages.hashCode();
+ + types.hashCode() + messages.hashCode() + props.hashCode();
}
/** Render this as <a href="http://json.org/">JSON</a>.*/
@@ -279,7 +356,7 @@ public class Protocol {
gen.writeStringField("namespace", namespace);
if (doc != null) gen.writeStringField("doc", doc);
-
+ props.write(gen);
gen.writeArrayFieldStart("types");
Schema.Names resolved = new Schema.Names(namespace);
for (Schema type : types.values())
@@ -339,6 +416,7 @@ public class Protocol {
parseTypes(json);
parseMessages(json);
parseDoc(json);
+ parseProps(json);
}
private void parseNamespace(JsonNode json) {
@@ -377,6 +455,17 @@ public class Protocol {
}
}
+ private void parseProps(JsonNode json) {
+ for (Iterator<String> i = json.getFieldNames(); i.hasNext();) {
+ String p = i.next(); // add non-reserved as props
+ if (!PROTOCOL_RESERVED.contains(p)) {
+ JsonNode prop = json.get(p);
+ if (prop.isValueNode() && prop.isTextual())
+ this.addProp(p,prop.getTextValue());
+ }
+ }
+ }
+
private void parseMessages(JsonNode json) {
JsonNode defs = json.get("messages");
if (defs == null) return; // no messages defined
@@ -389,6 +478,16 @@ public class Protocol {
private Message parseMessage(String messageName, JsonNode json) {
String doc = parseDocNode(json);
+ Map<String,String> mProps = new LinkedHashMap<String,String>();
+ for (Iterator<String> i = json.getFieldNames(); i.hasNext();) {
+ String p = i.next(); // add non-reserved as props
+ if (!MESSAGE_RESERVED.contains(p)) {
+ JsonNode prop = json.get(p);
+ if (prop.isValueNode() && prop.isTextual())
+ mProps.put(p,prop.getTextValue());
+ }
+ }
+
JsonNode requestNode = json.get("request");
if (requestNode == null || !requestNode.isArray())
throw new SchemaParseException("No request specified: "+json);
@@ -427,7 +526,7 @@ public class Protocol {
if (responseNode != null
&& Schema.parse(responseNode, types).getType() != Schema.Type.NULL)
throw new SchemaParseException("One way response must be null: "+json);
- return new Message(messageName, doc, request);
+ return new Message(messageName, doc, mProps, request);
}
Schema response = Schema.parse(responseNode, types);
@@ -448,7 +547,7 @@ public class Protocol {
}
}
- return new TwoWayMessage(messageName, doc, request, response,
+ return new TwoWayMessage(messageName, doc, mProps, request, response,
Schema.createUnion(errs));
}
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java Fri Dec 9 20:41:13 2011
@@ -114,7 +114,7 @@ public abstract class Schema {
}
}
- private static final class Props extends LinkedHashMap<String,String> {
+ static final class Props extends LinkedHashMap<String,String> {
private Set<String> reserved;
public Props(Set<String> reserved) {
super(1);
Modified: avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java (original)
+++ avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java Fri Dec 9 20:41:13 2011
@@ -342,6 +342,10 @@ public class SpecificCompiler {
Protocol newP = new Protocol(p.getName(), p.getDoc(), p.getNamespace());
Map<Schema,Schema> types = new LinkedHashMap<Schema,Schema>();
+ // Copy properties
+ for (Map.Entry<String,String> prop : p.getProps().entrySet())
+ newP.addProp(prop.getKey(), prop.getValue()); // copy props
+
// annotate types
Collection<Schema> namedTypes = new LinkedHashSet<Schema>();
for (Schema s : p.getTypes())
@@ -352,9 +356,9 @@ public class SpecificCompiler {
Map<String,Message> newM = newP.getMessages();
for (Message m : p.getMessages().values())
newM.put(m.getName(), m.isOneWay()
- ? newP.createMessage(m.getName(), m.getDoc(),
+ ? newP.createMessage(m.getName(), m.getDoc(), m.getProps(),
addStringType(m.getRequest(), types))
- : newP.createMessage(m.getName(), m.getDoc(),
+ : newP.createMessage(m.getName(), m.getDoc(), m.getProps(),
addStringType(m.getRequest(), types),
addStringType(m.getResponse(), types),
addStringType(m.getErrors(), types)));
Modified: avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj (original)
+++ avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj Fri Dec 9 20:41:13 2011
@@ -974,14 +974,12 @@ Protocol CompilationUnit():
/*
* Declaration syntax follows.
*/
-Schema NamedSchemaDeclaration():
+Schema NamedSchemaDeclaration(Map<String, JsonNode> props):
{
Schema s;
- Map<String, JsonNode> props = new LinkedHashMap<String, JsonNode>();
String savedSpace = this.namespace;
}
{
- ( SchemaProperty(props) )*
{
if (props.containsKey("namespace"))
this.namespace = getTextProp("namespace", props, token);
@@ -1048,6 +1046,11 @@ Protocol ProtocolDeclaration():
name = Identifier()
{
p = new Protocol(name, getDoc(), namespace);
+ for (String key : props.keySet())
+ if ("namespace".equals(key)) { // already handled: ignore
+ } else if (props.get(key).isTextual()) { // ignore other non-textual
+ p.addProp(key, getTextProp(key, props, token));
+ }
}
ProtocolBody(p)
{
@@ -1098,13 +1101,12 @@ void ProtocolBody(Protocol p):
Schema schema;
Message message;
Protocol importProtocol;
+ Map<String, JsonNode> props = new LinkedHashMap<String, JsonNode>();
}
{
- "{"
- (
- schema = NamedSchemaDeclaration()
-
- | <IMPORT>
+ "{"
+ (
+ <IMPORT>
((( importProtocol = ImportIdl() | importProtocol = ImportProtocol()) {
for (Schema s : importProtocol.getTypes())
names.put(s.getFullName(), s);
@@ -1112,10 +1114,15 @@ void ProtocolBody(Protocol p):
})
| schema = ImportSchema()
)
-
- | message = MessageDeclaration(p) {
- p.getMessages().put(message.getName(), message);
- }
+ |
+ ( SchemaProperty(props) )*
+ (
+ schema = NamedSchemaDeclaration(props)
+ |
+ message = MessageDeclaration(p, props) {
+ p.getMessages().put(message.getName(), message);
+ }
+ ) { props.clear(); }
) *
"}"
@@ -1289,7 +1296,7 @@ String MessageDocumentation():
}
}
-Message MessageDeclaration(Protocol p):
+Message MessageDeclaration(Protocol p, Map<String, JsonNode> props):
{
String msgDoc;
String name;
@@ -1298,6 +1305,7 @@ Message MessageDeclaration(Protocol p):
boolean oneWay = false;
List<Schema> errorSchemata = new ArrayList<Schema>();
errorSchemata.add(Protocol.SYSTEM_ERROR);
+ Map<String, String> propMap = new LinkedHashMap<String, String>();
}
{
msgDoc = MessageDocumentation()
@@ -1307,12 +1315,17 @@ Message MessageDeclaration(Protocol p):
[ "oneway" {oneWay = true; } | "throws" ErrorList(errorSchemata) ]
";"
{
+ for (String key : props.keySet())
+ if (props.get(key).isTextual()) { // ignore other non-textual
+ propMap.put(key, getTextProp(key, props, token));
+ }
+
Schema errors = Schema.createUnion(errorSchemata);
if (oneWay && response.getType() != Type.NULL)
throw error("One-way message'"+name+"' must return void", token);
return oneWay
- ? p.createMessage(name, msgDoc, request)
- : p.createMessage(name, msgDoc, request, response, errors);
+ ? p.createMessage(name, msgDoc, propMap, request)
+ : p.createMessage(name, msgDoc, propMap, request, response, errors);
}
}
Modified: avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl Fri Dec 9 20:41:13 2011
@@ -19,6 +19,7 @@
/**
* A simple test case.
*/
+@version("1.0.5")
@namespace("org.apache.avro.test")
protocol Simple {
/** A kind of record. */
@@ -55,6 +56,7 @@ protocol Simple {
string hello(string greeting);
TestRecord echo(TestRecord `record` = {"name": "bar"});
/** method 'add' takes @parameter 'arg1' @parameter 'arg2' */
+ @specialProp("test")
int add(int arg1, int arg2 = 0);
bytes echoBytes(bytes data);
void `error`() throws TestError;
Modified: avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr Fri Dec 9 20:41:13 2011
@@ -2,6 +2,7 @@
"protocol" : "Simple",
"namespace" : "org.apache.avro.test",
"doc" : "* A simple test case.",
+ "version" : "1.0.5",
"types" : [ {
"type" : "enum",
"name" : "Kind",
@@ -73,6 +74,7 @@
},
"add" : {
"doc" : "method 'add' takes @parameter 'arg1' @parameter 'arg2'",
+ "specialProp" : "test",
"request" : [ {
"name" : "arg1",
"type" : "int"
Modified: avro/trunk/share/test/schemas/simple.avpr
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/schemas/simple.avpr?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/share/test/schemas/simple.avpr (original)
+++ avro/trunk/share/test/schemas/simple.avpr Fri Dec 9 20:41:13 2011
@@ -1,6 +1,7 @@
{"namespace": "org.apache.avro.test",
"protocol": "Simple",
"doc": "Protocol used for testing.",
+ "version" : "1.6.2",
"types": [
{"name": "Kind", "type": "enum", "symbols": ["FOO","BAR","BAZ"]},
@@ -44,6 +45,7 @@
},
"add": {
+ "specialProp" : "test",
"request": [{"name": "arg1", "type": "int"}, {"name": "arg2", "type": "int"}],
"response": "int"
},
Modified: avro/trunk/share/test/schemas/social.avdl
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/schemas/social.avdl?rev=1212616&r1=1212615&r2=1212616&view=diff
==============================================================================
--- avro/trunk/share/test/schemas/social.avdl (original)
+++ avro/trunk/share/test/schemas/social.avdl Fri Dec 9 20:41:13 2011
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+@version("1.0.5")
@namespace("org.apache.avro.ipc.specific")
protocol Social {
record Person {