You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by jg...@apache.org on 2018/04/18 16:18:50 UTC
[kafka] branch trunk updated: KAFKA-3365;
Add documentation method for protocol types and update doc
generation (#4735)
This is an automated email from the ASF dual-hosted git repository.
jgus pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4623422 KAFKA-3365; Add documentation method for protocol types and update doc generation (#4735)
4623422 is described below
commit 462342210fd5d6f571d26b8af838fe77d5c5ca48
Author: Andras Beni <an...@cloudera.com>
AuthorDate: Wed Apr 18 18:18:45 2018 +0200
KAFKA-3365; Add documentation method for protocol types and update doc generation (#4735)
Reviewers: Sandor Murakozi <sm...@gmail.com>, Magnus Edenhill <ma...@edenhill.se>, Jason Gustafson <ja...@confluent.io>
---
build.gradle | 9 +-
.../kafka/common/protocol/types/ArrayOf.java | 22 ++-
.../apache/kafka/common/protocol/types/Schema.java | 2 -
.../apache/kafka/common/protocol/types/Type.java | 188 ++++++++++++++++++---
docs/protocol.html | 13 +-
5 files changed, 191 insertions(+), 43 deletions(-)
diff --git a/build.gradle b/build.gradle
index 8b07a95..8561eb4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -633,6 +633,13 @@ project(':core') {
standardOutput = new File(generatedDocsDir, "protocol_errors.html").newOutputStream()
}
+ task genProtocolTypesDocs(type: JavaExec) {
+ classpath = sourceSets.main.runtimeClasspath
+ main = 'org.apache.kafka.common.protocol.types.Type'
+ if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
+ standardOutput = new File(generatedDocsDir, "protocol_types.html").newOutputStream()
+ }
+
task genProtocolApiKeyDocs(type: JavaExec) {
classpath = sourceSets.main.runtimeClasspath
main = 'org.apache.kafka.common.protocol.ApiKeys'
@@ -696,7 +703,7 @@ project(':core') {
standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream()
}
- task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
+ task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs',
'genKafkaConfigDocs', 'genTopicConfigDocs',
':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
index 4213ecd..6609dfd 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
@@ -16,12 +16,16 @@
*/
package org.apache.kafka.common.protocol.types;
+import org.apache.kafka.common.protocol.types.Type.DocumentedType;
+
import java.nio.ByteBuffer;
/**
* Represents a type for an array of a particular type
*/
-public class ArrayOf extends Type {
+public class ArrayOf extends DocumentedType {
+
+ private static final String ARRAY_TYPE_NAME = "ARRAY";
private final Type type;
private final boolean nullable;
@@ -93,7 +97,7 @@ public class ArrayOf extends Type {
@Override
public String toString() {
- return "ARRAY(" + type + ")";
+ return ARRAY_TYPE_NAME + "(" + type + ")";
}
@Override
@@ -110,4 +114,18 @@ public class ArrayOf extends Type {
throw new SchemaException("Not an Object[].");
}
}
+
+ @Override
+ public String typeName() {
+ return ARRAY_TYPE_NAME;
+ }
+
+ @Override
+ public String documentation() {
+ return "Represents a sequence of objects of a given type T. " +
+ "Type T can be either a primitive type (e.g. " + STRING + ") or a structure. " +
+ "First, the length N is given as an " + INT32 + ". Then N instances of type T follow. " +
+ "A null array is represented with a length of -1. " +
+ "In protocol documentation an array of T instances is referred to as [T].";
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
index faa1540..cbcd449 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
@@ -193,6 +193,4 @@ public class Schema extends Type {
public void visit(ArrayOf array) {}
public void visit(Type field) {}
}
-
-
}
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
index 57d31f4..85916d5 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
@@ -64,13 +64,36 @@ public abstract class Type {
}
/**
+ * A Type that can return its description for documentation purposes.
+ */
+ public static abstract class DocumentedType extends Type {
+
+ /**
+ * Short name of the type to identify it in documentation;
+ * @return the name of the type
+ */
+ public abstract String typeName();
+
+ /**
+ * Documentation of the Type.
+ *
+ * @return details about valid values, representation
+ */
+ public abstract String documentation();
+
+ @Override
+ public String toString() {
+ return typeName();
+ }
+ }
+ /**
* The Boolean type represents a boolean value in a byte by using
* the value of 0 to represent false, and 1 to represent true.
*
* If for some reason a value that is not 0 or 1 is read,
* then any non-zero value will return true.
*/
- public static final Type BOOLEAN = new Type() {
+ public static final DocumentedType BOOLEAN = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
if ((Boolean) o)
@@ -91,7 +114,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "BOOLEAN";
}
@@ -102,9 +125,16 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a Boolean.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents a boolean value in a byte. " +
+ "Values 0 and 1 are used to represent false and true respectively. " +
+ "When reading a boolean value, any non-zero value is considered true.";
+ }
};
- public static final Type INT8 = new Type() {
+ public static final DocumentedType INT8 = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
buffer.put((Byte) o);
@@ -121,7 +151,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "INT8";
}
@@ -132,9 +162,14 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a Byte.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents an integer between -2<sup>7</sup> and 2<sup>7</sup>-1 inclusive.";
+ }
};
- public static final Type INT16 = new Type() {
+ public static final DocumentedType INT16 = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
buffer.putShort((Short) o);
@@ -151,7 +186,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "INT16";
}
@@ -162,9 +197,15 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a Short.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents an integer between -2<sup>15</sup> and 2<sup>15</sup>-1 inclusive. " +
+ "The values are encoded using two bytes in network byte order (big-endian).";
+ }
};
- public static final Type INT32 = new Type() {
+ public static final DocumentedType INT32 = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
buffer.putInt((Integer) o);
@@ -181,7 +222,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "INT32";
}
@@ -192,9 +233,15 @@ public abstract class Type {
else
throw new SchemaException(item + " is not an Integer.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents an integer between -2<sup>31</sup> and 2<sup>31</sup>-1 inclusive. " +
+ "The values are encoded using four bytes in network byte order (big-endian).";
+ }
};
- public static final Type UNSIGNED_INT32 = new Type() {
+ public static final DocumentedType UNSIGNED_INT32 = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
ByteUtils.writeUnsignedInt(buffer, (long) o);
@@ -211,7 +258,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "UINT32";
}
@@ -222,9 +269,15 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a Long.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents an integer between 0 and 2<sup>32</sup>-1 inclusive. " +
+ "The values are encoded using four bytes in network byte order (big-endian).";
+ }
};
- public static final Type INT64 = new Type() {
+ public static final DocumentedType INT64 = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
buffer.putLong((Long) o);
@@ -241,7 +294,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "INT64";
}
@@ -252,9 +305,15 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a Long.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents an integer between -2<sup>63</sup> and 2<sup>63</sup>-1 inclusive. " +
+ "The values are encoded using eight bytes in network byte order (big-endian).";
+ }
};
- public static final Type STRING = new Type() {
+ public static final DocumentedType STRING = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
byte[] bytes = Utils.utf8((String) o);
@@ -282,7 +341,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "STRING";
}
@@ -293,9 +352,16 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a String.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents a sequence of characters. First the length N is given as an " + INT16 +
+ ". Then N bytes follow which are the UTF-8 encoding of the character sequence. " +
+ "Length must not be negative.";
+ }
};
- public static final Type NULLABLE_STRING = new Type() {
+ public static final DocumentedType NULLABLE_STRING = new DocumentedType() {
@Override
public boolean isNullable() {
return true;
@@ -336,7 +402,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "NULLABLE_STRING";
}
@@ -350,9 +416,16 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a String.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents a sequence of characters or null. For non-null strings, first the length N is given as an " + INT16 +
+ ". Then N bytes follow which are the UTF-8 encoding of the character sequence. " +
+ "A null value is encoded with length of -1 and there are no following bytes.";
+ }
};
- public static final Type BYTES = new Type() {
+ public static final DocumentedType BYTES = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
ByteBuffer arg = (ByteBuffer) o;
@@ -383,7 +456,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "BYTES";
}
@@ -394,9 +467,15 @@ public abstract class Type {
else
throw new SchemaException(item + " is not a java.nio.ByteBuffer.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents a raw sequence of bytes. First the length N is given as an " + INT32 +
+ ". Then N bytes follow.";
+ }
};
- public static final Type NULLABLE_BYTES = new Type() {
+ public static final DocumentedType NULLABLE_BYTES = new DocumentedType() {
@Override
public boolean isNullable() {
return true;
@@ -440,7 +519,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "NULLABLE_BYTES";
}
@@ -454,9 +533,15 @@ public abstract class Type {
throw new SchemaException(item + " is not a java.nio.ByteBuffer.");
}
+
+ @Override
+ public String documentation() {
+ return "Represents a raw sequence of bytes or null. For non-null values, first the length N is given as an " + INT32 +
+ ". Then N bytes follow. A null value is encoded with length of -1 and there are no following bytes.";
+ }
};
- public static final Type RECORDS = new Type() {
+ public static final DocumentedType RECORDS = new DocumentedType() {
@Override
public boolean isNullable() {
return true;
@@ -486,7 +571,7 @@ public abstract class Type {
}
@Override
- public String toString() {
+ public String typeName() {
return "RECORDS";
}
@@ -500,9 +585,16 @@ public abstract class Type {
throw new SchemaException(item + " is not an instance of " + Records.class.getName());
}
+
+ @Override
+ public String documentation() {
+ return "Represents a sequence of Kafka records as " + NULLABLE_BYTES + ". " +
+ "For a detailed description of records see " +
+ "<a href=\"/documentation/#messageformat\">Message Sets</a>.";
+ }
};
- public static final Type VARINT = new Type() {
+ public static final DocumentedType VARINT = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
ByteUtils.writeVarint((Integer) o, buffer);
@@ -520,7 +612,7 @@ public abstract class Type {
throw new SchemaException(item + " is not an integer");
}
- public String toString() {
+ public String typeName() {
return "VARINT";
}
@@ -528,9 +620,16 @@ public abstract class Type {
public int sizeOf(Object o) {
return ByteUtils.sizeOfVarint((Integer) o);
}
+
+ @Override
+ public String documentation() {
+ return "Represents an integer between -2<sup>31</sup> and 2<sup>31</sup>-1 inclusive. " +
+ "Encoding follows the variable-length zig-zag encoding from " +
+ " <a href=\"http://code.google.com/apis/protocolbuffers/docs/encoding.html\"> Google Protocol Buffers</a>.";
+ }
};
- public static final Type VARLONG = new Type() {
+ public static final DocumentedType VARLONG = new DocumentedType() {
@Override
public void write(ByteBuffer buffer, Object o) {
ByteUtils.writeVarlong((Long) o, buffer);
@@ -548,7 +647,7 @@ public abstract class Type {
throw new SchemaException(item + " is not a long");
}
- public String toString() {
+ public String typeName() {
return "VARLONG";
}
@@ -556,6 +655,43 @@ public abstract class Type {
public int sizeOf(Object o) {
return ByteUtils.sizeOfVarlong((Long) o);
}
+
+ @Override
+ public String documentation() {
+ return "Represents an integer between -2<sup>63</sup> and 2<sup>63</sup>-1 inclusive. " +
+ "Encoding follows the variable-length zig-zag encoding from " +
+ " <a href=\"http://code.google.com/apis/protocolbuffers/docs/encoding.html\"> Google Protocol Buffers</a>.";
+ }
};
+ private static String toHtml() {
+
+ DocumentedType[] types = {
+ BOOLEAN, INT8, INT16, INT32, INT64,
+ UNSIGNED_INT32, VARINT, VARLONG,
+ STRING, NULLABLE_STRING, BYTES, NULLABLE_BYTES,
+ RECORDS, new ArrayOf(STRING)};
+ final StringBuilder b = new StringBuilder();
+ b.append("<table class=\"data-table\"><tbody>\n");
+ b.append("<tr>");
+ b.append("<th>Type</th>\n");
+ b.append("<th>Description</th>\n");
+ b.append("</tr>\n");
+ for (DocumentedType type : types) {
+ b.append("<tr>");
+ b.append("<td>");
+ b.append(type.typeName());
+ b.append("</td>");
+ b.append("<td>");
+ b.append(type.documentation());
+ b.append("</td>");
+ b.append("</tr>\n");
+ }
+ b.append("</table>\n");
+ return b.toString();
+ }
+
+ public static void main(String[] args) {
+ System.out.println(toHtml());
+ }
}
diff --git a/docs/protocol.html b/docs/protocol.html
index 0daa3b8..416823a 100644
--- a/docs/protocol.html
+++ b/docs/protocol.html
@@ -163,18 +163,7 @@ Kafka request. SASL/GSSAPI authentication is performed starting with this packet
<h5><a id="protocol_types" href="#protocol_types">Protocol Primitive Types</a></h5>
<p>The protocol is built out of the following primitive types.</p>
-
-<p><b>Fixed Width Primitives</b><p>
-
-<p>int8, int16, int32, int64 - Signed integers with the given precision (in bits) stored in big endian order.</p>
-
-<p><b>Variable Length Primitives</b><p>
-
-<p>bytes, string - These types consist of a signed integer giving a length N followed by N bytes of content. A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32.</p>
-
-<p><b>Arrays</b><p>
-
-<p>This is a notation for handling repeated structures. These will always be encoded as an int32 size containing the length N followed by N repetitions of the structure which can itself be made up of other primitive types. In the BNF grammars below we will show an array of a structure foo as [foo].</p>
+<!--#include virtual="generated/protocol_types.html" -->
<h5><a id="protocol_grammar" href="#protocol_grammar">Notes on reading the request format grammars</a></h5>
--
To stop receiving notification emails like this one, please contact
jgus@apache.org.