You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/02/26 00:34:18 UTC
[1/4] storm git commit: [STORM-1540] Fix Debug/Sampling for Trident
Repository: storm
Updated Branches:
refs/heads/master 8fa5aa537 -> c193a90c5
[STORM-1540] Fix Debug/Sampling for Trident
When ConsList emitted by a trident spout has to be transferred over
the network, it fails during Serialization. The proposed fix is to make ConsList
kyro serializable.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0f0dea0b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0f0dea0b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0f0dea0b
Branch: refs/heads/master
Commit: 0f0dea0b4d2b17d6e8dc32e82341b8aec70ef6e9
Parents: 12ceb09
Author: Arun Mahadevan <ai...@hortonworks.com>
Authored: Tue Feb 16 15:55:25 2016 +0530
Committer: Arun Mahadevan <ai...@hortonworks.com>
Committed: Tue Feb 16 16:07:27 2016 +0530
----------------------------------------------------------------------
.../serialization/SerializationFactory.java | 2 ++
.../apache/storm/trident/tuple/ConsList.java | 20 ++++++++++++++++++--
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0f0dea0b/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java b/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
index 4007138..8415ce3 100644
--- a/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
+++ b/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
@@ -24,6 +24,7 @@ import org.apache.storm.serialization.types.ArrayListSerializer;
import org.apache.storm.serialization.types.HashMapSerializer;
import org.apache.storm.serialization.types.HashSetSerializer;
import org.apache.storm.transactional.TransactionAttempt;
+import org.apache.storm.trident.tuple.ConsList;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.ListDelegate;
import org.apache.storm.utils.Utils;
@@ -68,6 +69,7 @@ public class SerializationFactory {
k.register(Values.class);
k.register(org.apache.storm.metric.api.IMetricsConsumer.DataPoint.class);
k.register(org.apache.storm.metric.api.IMetricsConsumer.TaskInfo.class);
+ k.register(ConsList.class);
try {
JavaBridge.registerPrimitives(k);
JavaBridge.registerCollections(k);
http://git-wip-us.apache.org/repos/asf/storm/blob/0f0dea0b/storm-core/src/jvm/org/apache/storm/trident/tuple/ConsList.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/trident/tuple/ConsList.java b/storm-core/src/jvm/org/apache/storm/trident/tuple/ConsList.java
index 55c1e79..cef2cc4 100644
--- a/storm-core/src/jvm/org/apache/storm/trident/tuple/ConsList.java
+++ b/storm-core/src/jvm/org/apache/storm/trident/tuple/ConsList.java
@@ -18,12 +18,18 @@
package org.apache.storm.trident.tuple;
import java.util.AbstractList;
+import java.util.ArrayList;
import java.util.List;
public class ConsList extends AbstractList<Object> {
List<Object> _elems;
Object _first;
-
+
+ // for kryo
+ private ConsList() {
+ _elems = new ArrayList<>();
+ }
+
public ConsList(Object o, List<Object> elems) {
_elems = elems;
_first = o;
@@ -39,6 +45,16 @@ public class ConsList extends AbstractList<Object> {
@Override
public int size() {
- return _elems.size() + 1;
+ return _first == null ? _elems.size() : _elems.size() + 1;
+ }
+
+ // for kryo
+ @Override
+ public void add(int index, Object element) {
+ if (index == 0) {
+ _first = element;
+ } else {
+ _elems.add(index - 1, element);
+ }
}
}
[2/4] storm git commit: Merge branch 'STORM-1540-new' of
https://github.com/arunmahadevan/storm into STORM-1540
Posted by ka...@apache.org.
Merge branch 'STORM-1540-new' of https://github.com/arunmahadevan/storm into STORM-1540
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cdf139bf
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cdf139bf
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cdf139bf
Branch: refs/heads/master
Commit: cdf139bf0a3a7c18552208407198b39112a26a72
Parents: 838ae33 0f0dea0
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Feb 25 15:04:53 2016 -0800
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Feb 25 15:04:53 2016 -0800
----------------------------------------------------------------------
.../serialization/SerializationFactory.java | 2 ++
.../apache/storm/trident/tuple/ConsList.java | 20 ++++++++++++++++++--
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[3/4] storm git commit: add STORM-1540 to CHANGELOG.md
Posted by ka...@apache.org.
add STORM-1540 to CHANGELOG.md
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/73959526
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/73959526
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/73959526
Branch: refs/heads/master
Commit: 73959526c324da1a72aaac8b48e528a9e2118c8d
Parents: cdf139b
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Feb 25 15:12:46 2016 -0800
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Feb 25 15:12:46 2016 -0800
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/73959526/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fc81653..ca8873b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,7 @@
* STORM-1521: When using Kerberos login from keytab with multiple bolts/executors ticket is not renewed in hbase bolt.
## 1.0.0
+ * STORM-1540: Fix Debug/Sampling for Trident
* STORM-1522: REST API throws invalid worker log links
* STORM-1541: Change scope of 'hadoop-minicluster' to test
* STORM-1532: Fix readCommandLineOpts to parse JSON correctly in windows
[4/4] storm git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/storm
Posted by ka...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c193a90c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c193a90c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c193a90c
Branch: refs/heads/master
Commit: c193a90c543ec3236391edc0cc133e45efcfa72a
Parents: 7395952 8fa5aa5
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Feb 25 15:34:09 2016 -0800
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Feb 25 15:34:09 2016 -0800
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/nimbus.clj | 6 ++-
.../src/clj/org/apache/storm/daemon/worker.clj | 8 ++--
.../storm/cluster/IStormClusterState.java | 2 +
.../storm/cluster/StormClusterStateImpl.java | 8 +++-
.../storm/utils/WorkerBackpressureCallback.java | 2 +-
.../storm/utils/WorkerBackpressureThread.java | 38 +++++++++++----
.../utils/WorkerBackpressureThreadTest.java | 50 ++++++++++++++++++++
7 files changed, 99 insertions(+), 15 deletions(-)
----------------------------------------------------------------------