You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2017/05/19 21:25:56 UTC
[1/3] storm git commit: STORM-2499: Add Serialization plugin for
EventHub System Properties
Repository: storm
Updated Branches:
refs/heads/master 64e29f365 -> e33c415e9
STORM-2499: Add Serialization plugin for EventHub System Properties
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ca17c4ff
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ca17c4ff
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ca17c4ff
Branch: refs/heads/master
Commit: ca17c4ff10231a5d93deb3d4ac934140ccec674d
Parents: 9755ff5
Author: Ranjan Banerjee <ra...@microsoft.com>
Authored: Wed May 3 17:39:31 2017 -0700
Committer: Ranjan Banerjee <ra...@microsoft.com>
Committed: Tue May 9 16:12:14 2017 -0700
----------------------------------------------------------------------
external/storm-eventhubs/README.md | 4 ++++
.../eventhubs/spout/BinaryEventDataScheme.java | 17 ++++++++++-------
.../storm/eventhubs/spout/EventDataScheme.java | 6 ++----
.../storm/eventhubs/spout/FieldConstants.java | 1 +
4 files changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ca17c4ff/external/storm-eventhubs/README.md
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/README.md b/external/storm-eventhubs/README.md
index 1586ca8..681ab2d 100755
--- a/external/storm-eventhubs/README.md
+++ b/external/storm-eventhubs/README.md
@@ -32,6 +32,10 @@ We have included a simple EventHubs send client for testing purpose. You can run
[username] [password] [entityPath] [partitionId] [messageSize] [messageCount]
If you want to send messages to all partitions, use "-1" as partitionId.
+### Serialization Scheme ###
+ By default the serialization scheme is StringEventDataScheme where only the body of the eventdata is being sent.
+ To have a more comprehensive Eventhub metadata exposure look into the BinaryEventDataScheme.
+
### Windows Azure Eventhubs ###
http://azure.microsoft.com/en-us/services/event-hubs/
http://git-wip-us.apache.org/repos/asf/storm/blob/ca17c4ff/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
index 99d764c..5c767e8 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
@@ -31,9 +31,11 @@ import java.util.Map;
/**
* An Event Data Scheme which deserializes message payload into the raw bytes.
*
- * The resulting tuple would contain two items, the first being the message
+ * The resulting tuple would contain three items, the first being the message
* bytes, and the second a map of properties that include metadata, which can be
- * used to determine who processes the message, and how it is processed.
+ * used to determine who processes the message, and how it is processed.The third is
+ * the system properties which exposes information like enqueue-time, offset and
+ * sequence number
*/
public class BinaryEventDataScheme implements IEventDataScheme {
@@ -56,16 +58,17 @@ public class BinaryEventDataScheme implements IEventDataScheme {
throw new RuntimeException(e);
}
}
- Map metaDataMap = eventData.getProperties().size() > 0 ? eventData.getProperties() : null;
+ Map metaDataMap = eventData.getProperties();
+ Map systemMetaDataMap = eventData.getSystemProperties();
fieldContents.add(messageData);
- if ( metaDataMap != null ) {
- fieldContents.add(metaDataMap);
- }
+ fieldContents.add(metaDataMap);
+ fieldContents.add(systemMetaDataMap);
return fieldContents;
}
@Override
public Fields getOutputFields() {
- return new Fields(FieldConstants.Message, FieldConstants.META_DATA);
+ return new Fields(FieldConstants.Message, FieldConstants.META_DATA,
+ FieldConstants.SYSTEM_META_DATA);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/ca17c4ff/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
index a537975..96bd6d0 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
@@ -65,11 +65,9 @@ public class EventDataScheme implements IEventDataScheme {
throw e;
}
}
- Map metaDataMap = eventData.getProperties().size() > 0 ? eventData.getProperties() : null;
+ Map metaDataMap = eventData.getProperties();
fieldContents.add(messageData);
- if ( metaDataMap != null ) {
- fieldContents.add(metaDataMap);
- }
+ fieldContents.add(metaDataMap);
return fieldContents;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/ca17c4ff/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
index 079d1da..4c1c3e7 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
@@ -23,5 +23,6 @@ public class FieldConstants {
public static final String Offset = "offset";
public static final String Message = "message";
public static final String META_DATA = "metadata";
+ public static final String SYSTEM_META_DATA = "eventdata_system_properties";
public static final String DefaultStartingOffset = "-1";
}
[3/3] storm git commit: Added STORM-2499 to CHANGELOG.
Posted by sr...@apache.org.
Added STORM-2499 to CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e33c415e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e33c415e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e33c415e
Branch: refs/heads/master
Commit: e33c415e9cba89b6282b70af5c7f285071f62dfb
Parents: a8cf365
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Fri May 19 13:52:45 2017 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Fri May 19 13:52:45 2017 -0700
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e33c415e/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 024f861..b2ed259 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-2499: Add Serialization plugin for EventHub System Properties
* STORM-2520: AutoHDFS should prefer cluster-wise hdfs kerberos principal
* STORM-2519: Modify AbstractAutoCreds to look for configKeys in both nimbus and topology configs
* STORM-2493: update documents to reflect the changes
[2/3] storm git commit: Merge branch 'eventhub4' of
https://github.com/rban1/storm into STORM-2499
Posted by sr...@apache.org.
Merge branch 'eventhub4' of https://github.com/rban1/storm into STORM-2499
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a8cf3653
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a8cf3653
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a8cf3653
Branch: refs/heads/master
Commit: a8cf3653bafb6b0f8b7c0e82e39ea619257671b3
Parents: 64e29f3 ca17c4f
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Fri May 19 13:51:40 2017 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Fri May 19 13:51:40 2017 -0700
----------------------------------------------------------------------
external/storm-eventhubs/README.md | 4 ++++
.../eventhubs/spout/BinaryEventDataScheme.java | 17 ++++++++++-------
.../storm/eventhubs/spout/EventDataScheme.java | 6 ++----
.../storm/eventhubs/spout/FieldConstants.java | 1 +
4 files changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------