You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/02/07 02:27:35 UTC
[01/13] storm git commit: STORM-2338: Subprocess exception handling
is broken in storm.py on Windows environment
Repository: storm
Updated Branches:
refs/heads/1.0.x-branch a4e20fa85 -> 452a37ff7
STORM-2338: Subprocess exception handling is broken in storm.py on Windows environment
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1b43804d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1b43804d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1b43804d
Branch: refs/heads/1.0.x-branch
Commit: 1b43804ddcd4f578b96b90db4acd058d7514106e
Parents: 8f6dd29
Author: Tibor Kiss <ti...@gmail.com>
Authored: Fri Feb 3 09:10:10 2017 +0100
Committer: Tibor Kiss <ti...@gmail.com>
Committed: Fri Feb 3 09:10:10 2017 +0100
----------------------------------------------------------------------
bin/storm.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1b43804d/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index 6bb5a1b..feb6c7a 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -218,7 +218,7 @@ def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[]
try:
ret = sub.check_output(all_args, stderr=sub.STDOUT)
print(ret)
- except sub.CalledProcessor as e:
+ except sub.CalledProcessError as e:
sys.exit(e.returncode)
else:
os.execvp(JAVA_CMD, all_args)
[03/13] storm git commit: Merge branch 'STORM-2197-1.0.x-merge' into
1.0.x-branch
Posted by ka...@apache.org.
Merge branch 'STORM-2197-1.0.x-merge' into 1.0.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/07ba34f0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/07ba34f0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/07ba34f0
Branch: refs/heads/1.0.x-branch
Commit: 07ba34f0c53ea6ff0389cc1b57b933943ddae9d1
Parents: a4e20fa 6c9d072
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 7 10:36:04 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:36:04 2017 +0900
----------------------------------------------------------------------
.../org/apache/storm/security/auth/ThriftClient.java | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[08/13] storm git commit: STORM-2335 Fix broken Topology
visualization with empty ':transferred' in executor stats
Posted by ka...@apache.org.
STORM-2335 Fix broken Topology visualization with empty ':transferred' in executor stats
* also fix another bug: stream table and graph box was created for each 'Show Visualization' click
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a22793c9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a22793c9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a22793c9
Branch: refs/heads/1.0.x-branch
Commit: a22793c901afd8a7e5e3683cb58d414cdde8d16f
Parents: ec39ca1
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Wed Feb 1 17:17:46 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:40:04 2017 +0900
----------------------------------------------------------------------
storm-core/src/ui/public/js/visualization.js | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a22793c9/storm-core/src/ui/public/js/visualization.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/visualization.js b/storm-core/src/ui/public/js/visualization.js
index 4daadc2..e82801f 100644
--- a/storm-core/src/ui/public/js/visualization.js
+++ b/storm-core/src/ui/public/js/visualization.js
@@ -309,7 +309,7 @@ function gather_stream_count(stats, stream, time) {
var transferred = 0;
if(stats)
for(var i = 0; i < stats.length; i++) {
- if(stats[i][":transferred"] != null)
+ if(stats[i][":transferred"] != null && stats[i][":transferred"][time] != undefined)
{
var stream_trans = stats[i][":transferred"][time][stream];
if(stream_trans != null)
@@ -391,12 +391,14 @@ function show_visualization(sys) {
getStatic("/templates/topology-page-template.html", function(template) {
jsError(function() {
var topologyVisualization = $("#visualization-container");
- topologyVisualization.append(
- Mustache.render($(template)
- .filter("#topology-visualization-container-template")
- .html(),
- response));
- });
+ if (topologyVisualization.find("canvas").length == 0) {
+ topologyVisualization.append(
+ Mustache.render($(template)
+ .filter("#topology-visualization-container-template")
+ .html(),
+ response));
+ }
+ });
if(sys == null)
{
[02/13] storm git commit: STORM-2197: Fix for NimbusClient connectins
leak due to ThriftClient connection's leakage in case of errors.
Posted by ka...@apache.org.
STORM-2197: Fix for NimbusClient connectins leak due to ThriftClient connection's leakage in case of errors.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6c9d0723
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6c9d0723
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6c9d0723
Branch: refs/heads/1.0.x-branch
Commit: 6c9d072375496bb2542db9ba664cb19b204dad5a
Parents: a4e20fa
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Thu Nov 10 09:35:31 2016 +0530
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:35:56 2017 +0900
----------------------------------------------------------------------
.../org/apache/storm/security/auth/ThriftClient.java | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6c9d0723/storm-core/src/jvm/org/apache/storm/security/auth/ThriftClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/security/auth/ThriftClient.java b/storm-core/src/jvm/org/apache/storm/security/auth/ThriftClient.java
index bcc1a11..be062f4 100644
--- a/storm-core/src/jvm/org/apache/storm/security/auth/ThriftClient.java
+++ b/storm-core/src/jvm/org/apache/storm/security/auth/ThriftClient.java
@@ -74,9 +74,10 @@ public class ThriftClient implements AutoCloseable {
}
public synchronized void reconnect() {
- close();
+ close();
+ TSocket socket = null;
try {
- TSocket socket = new TSocket(_host, _port);
+ socket = new TSocket(_host, _port);
if(_timeout!=null) {
socket.setTimeout(_timeout);
}
@@ -97,7 +98,13 @@ public class ThriftClient implements AutoCloseable {
Utils.getInt(_conf.get(Config.STORM_NIMBUS_RETRY_INTERVAL_CEILING)),
_retryForever);
_transport = connectionRetry.doConnectWithRetry(transportPlugin, socket, _host, _asUser);
- } catch (IOException ex) {
+ } catch (Exception ex) {
+ // close the socket, which releases connection if it has created any.
+ if(socket != null) {
+ try {
+ socket.close();
+ } catch (Exception e) {}
+ }
throw new RuntimeException(ex);
}
_protocol = null;
[10/13] storm git commit: STORM-2236 Close Localizer and
AsyncLocalizer when supervisor is shutting down
Posted by ka...@apache.org.
STORM-2236 Close Localizer and AsyncLocalizer when supervisor is shutting down
* this patch gets rid of alive non-daemon threads which prevent JVM process shutdown
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/64a9dd0b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/64a9dd0b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/64a9dd0b
Branch: refs/heads/1.0.x-branch
Commit: 64a9dd0b80bd7be9b9d45d7bf508e7d81ce2328b
Parents: 59ee79d
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Wed Feb 1 17:45:54 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:40:59 2017 +0900
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/64a9dd0b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
index 59a3e6e..a6adace 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -77,7 +77,7 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
private final StormTimer eventTimer;
private final StormTimer blobUpdateTimer;
private final Localizer localizer;
- private final ILocalizer asyncLocalizer;
+ private final AsyncLocalizer asyncLocalizer;
private EventManager eventManager;
private ReadClusterState readState;
@@ -279,6 +279,8 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
if (readState != null) {
readState.close();
}
+ asyncLocalizer.shutdown();
+ localizer.shutdown();
getStormClusterState().disconnect();
} catch (Exception e) {
LOG.error("Error Shutting down", e);
[04/13] storm git commit: STORM-2321 Handle blobstore zk key deletion
in KeySequenceNumber
Posted by ka...@apache.org.
STORM-2321 Handle blobstore zk key deletion in KeySequenceNumber
* If NoNodeException is thrown in getKeySequenceNumber, treat it as KeyNotFoundException
* Change callers to handle KeyNotFoundException accordingly
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3b12349c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3b12349c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3b12349c
Branch: refs/heads/1.0.x-branch
Commit: 3b12349c97242bffb00a6e97314c8abb2aed7967
Parents: 07ba34f
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Jan 31 17:00:31 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:37:40 2017 +0900
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/nimbus.clj | 6 +++++-
.../apache/storm/blobstore/BlobStoreUtils.java | 6 +++++-
.../storm/blobstore/BlobSynchronizer.java | 13 +++++++++---
.../storm/blobstore/KeySequenceNumber.java | 21 ++++++++++++++------
.../storm/blobstore/LocalFsBlobStore.java | 4 +++-
5 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3b12349c/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
index daccb83..7a91ede 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
@@ -1190,7 +1190,11 @@
(.deleteBlob blob-store key nimbus-subject))
(log-debug "Creating list of key entries for blobstore inside zookeeper" all-keys "local" locally-available-active-keys)
(doseq [key locally-available-active-keys]
- (.setup-blobstore! storm-cluster-state key (:nimbus-host-port-info nimbus) (get-version-for-key key nimbus-host-port-info conf)))))
+ (try
+ (.setup-blobstore! storm-cluster-state key (:nimbus-host-port-info nimbus) (get-version-for-key key nimbus-host-port-info conf))
+ (catch KeyNotFoundException _
+ ; invalid key, remove it from blobstore
+ (.deleteBlob blob-store key nimbus-subject))))))
(defn- get-errors [storm-cluster-state storm-id component-id]
(->> (.errors storm-cluster-state storm-id component-id)
http://git-wip-us.apache.org/repos/asf/storm/blob/3b12349c/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
index 071c4f5..473984a 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
@@ -29,6 +29,7 @@ import org.apache.storm.utils.Utils;
import org.apache.storm.utils.ZookeeperAuthInfo;
import org.apache.curator.framework.CuratorFramework;
import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -238,7 +239,7 @@ public class BlobStoreUtils {
LOG.debug("StateInfo for update {}", stateInfo);
Set<NimbusInfo> nimbusInfoList = getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
- for (NimbusInfo nimbusInfo:nimbusInfoList) {
+ for (NimbusInfo nimbusInfo : nimbusInfoList) {
if (nimbusInfo.getHost().equals(nimbusDetails.getHost())) {
isListContainsCurrentNimbusInfo = true;
break;
@@ -249,6 +250,9 @@ public class BlobStoreUtils {
LOG.debug("Updating state inside zookeeper for an update");
createStateInZookeeper(conf, key, nimbusDetails);
}
+ } catch (KeeperException.NoNodeException | KeyNotFoundException e) {
+ //race condition with a delete
+ return;
} catch (Exception exp) {
throw new RuntimeException(exp);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3b12349c/storm-core/src/jvm/org/apache/storm/blobstore/BlobSynchronizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/BlobSynchronizer.java b/storm-core/src/jvm/org/apache/storm/blobstore/BlobSynchronizer.java
index bd34c32..c6647ac 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/BlobSynchronizer.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/BlobSynchronizer.java
@@ -17,6 +17,7 @@
*/
package org.apache.storm.blobstore;
+import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.nimbus.NimbusInfo;
import org.apache.curator.framework.CuratorFramework;
import org.slf4j.Logger;
@@ -78,9 +79,15 @@ public class BlobSynchronizer {
LOG.debug("Key set Blobstore-> Zookeeper-> DownloadSet {}-> {}-> {}", getBlobStoreKeySet(), getZookeeperKeySet(), keySetToDownload);
for (String key : keySetToDownload) {
- Set<NimbusInfo> nimbusInfoSet = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
- if(BlobStoreUtils.downloadMissingBlob(conf, blobStore, key, nimbusInfoSet)) {
- BlobStoreUtils.createStateInZookeeper(conf, key, nimbusInfo);
+ try {
+ Set<NimbusInfo> nimbusInfoSet = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
+ if (BlobStoreUtils.downloadMissingBlob(conf, blobStore, key, nimbusInfoSet)) {
+ BlobStoreUtils.createStateInZookeeper(conf, key, nimbusInfo);
+ }
+ } catch (KeyNotFoundException e) {
+ LOG.debug("Detected deletion for the key {} - deleting the blob instead", key);
+ // race condition with a delete, delete the blob in key instead
+ blobStore.deleteBlob(key, BlobStoreUtils.getNimbusSubject());
}
}
if (zkClient !=null) {
http://git-wip-us.apache.org/repos/asf/storm/blob/3b12349c/storm-core/src/jvm/org/apache/storm/blobstore/KeySequenceNumber.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/KeySequenceNumber.java b/storm-core/src/jvm/org/apache/storm/blobstore/KeySequenceNumber.java
index 2425761..adbd4c4 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/KeySequenceNumber.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/KeySequenceNumber.java
@@ -18,10 +18,12 @@
package org.apache.storm.blobstore;
+import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.nimbus.NimbusInfo;
import org.apache.storm.utils.Utils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,12 +132,12 @@ public class KeySequenceNumber {
this.nimbusInfo = nimbusInfo;
}
- public synchronized int getKeySequenceNumber(Map conf) {
+ public synchronized int getKeySequenceNumber(Map conf) throws KeyNotFoundException {
TreeSet<Integer> sequenceNumbers = new TreeSet<Integer>();
CuratorFramework zkClient = BlobStoreUtils.createZKClient(conf);
try {
// Key has not been created yet and it is the first time it is being created
- if(zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + "/" + key) == null) {
+ if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + "/" + key) == null) {
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT)
.withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE).forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key);
zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key,
@@ -148,7 +150,7 @@ public class KeySequenceNumber {
// if all go down which is unlikely. Hence there might be a need to update the blob if all go down.
List<String> stateInfoList = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key);
LOG.debug("stateInfoList-size {} stateInfoList-data {}", stateInfoList.size(), stateInfoList);
- if(stateInfoList.isEmpty()) {
+ if (stateInfoList.isEmpty()) {
return getMaxSequenceNumber(zkClient);
}
@@ -156,7 +158,7 @@ public class KeySequenceNumber {
// In all other cases check for the latest update sequence of the blob on the nimbus
// and assign the appropriate number. Check if all are have same sequence number,
// if not assign the highest sequence number.
- for (String stateInfo:stateInfoList) {
+ for (String stateInfo : stateInfoList) {
sequenceNumbers.add(Integer.parseInt(BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo(stateInfo)
.getSequenceNumber()));
}
@@ -195,15 +197,22 @@ public class KeySequenceNumber {
return sequenceNumbers.first() + 1;
}
}
+
+ // Normal create update sync scenario returns the greatest sequence number in the set
+ return sequenceNumbers.last();
+ } catch (KeeperException.NoNodeException e) {
+ // there's a race condition with a delete: either blobstore or blobstoremaxsequence
+ // this should be thrown to the caller to indicate that the key is invalid now
+ throw new KeyNotFoundException(key);
} catch(Exception e) {
+ // in other case, just set this to 0 to trigger re-sync later
LOG.error("Exception {}", e);
+ return INITIAL_SEQUENCE_NUMBER - 1;
} finally {
if (zkClient != null) {
zkClient.close();
}
}
- // Normal create update sync scenario returns the greatest sequence number in the set
- return sequenceNumbers.last();
}
private boolean checkIfStateContainsCurrentNimbusHost(List<String> stateInfoList, NimbusInfo nimbusInfo) {
http://git-wip-us.apache.org/repos/asf/storm/blob/3b12349c/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java b/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
index 345d591..648f5cd 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
@@ -281,7 +281,7 @@ public class LocalFsBlobStore extends BlobStore {
}
//This additional check and download is for nimbus high availability in case you have more than one nimbus
- public synchronized boolean checkForBlobOrDownload(String key) {
+ public synchronized boolean checkForBlobOrDownload(String key) throws KeyNotFoundException {
boolean checkBlobDownload = false;
try {
List<String> keyList = BlobStoreUtils.getKeyListFromBlobStore(this);
@@ -295,6 +295,8 @@ public class LocalFsBlobStore extends BlobStore {
}
}
}
+ } catch (KeyNotFoundException e) {
+ throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
[13/13] storm git commit: STORM-2338, STORM-2197, STORM-2324,
STORM-2321, STORM-2326, STORM-2335: CHANGELOG
Posted by ka...@apache.org.
STORM-2338, STORM-2197, STORM-2324, STORM-2321, STORM-2326, STORM-2335: CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/452a37ff
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/452a37ff
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/452a37ff
Branch: refs/heads/1.0.x-branch
Commit: 452a37ff7372f69034329ebc93ffd2936279dbc8
Parents: 00b633d
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 7 11:27:02 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 11:27:02 2017 +0900
----------------------------------------------------------------------
CHANGELOG.md | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/452a37ff/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 58cfd16..50878fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,10 @@
## 1.0.3
+ * STORM-2197: NimbusClient connectins leak due to leakage in ThriftClient
+ * STORM-2321: Handle blobstore zk key deletion in KeySequenceNumber.
+ * STORM-2324: Fix deployment failure if resources directory is missing in topology jar
+ * STORM-2335: Fix broken Topology visualization with empty ':transferred' in executor stats
+ * STORM-2336: Close Localizer and AsyncLocalizer when supervisor is shutting down
+ * STORM-2338: Subprocess exception handling is broken in storm.py on Windows environment
* STORM-2337: Broken documentation generation for storm-metrics-profiling-internal-actions.md and windows-users-guide.md
* STORM-2325: Logviewer doesn't consider 'storm.local.hostname'
* STORM-1742: More accurate 'complete latency'
[07/13] storm git commit: Merge branch 'STORM-2324-1.0.x-merge' into
1.0.x-branch
Posted by ka...@apache.org.
Merge branch 'STORM-2324-1.0.x-merge' into 1.0.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ec39ca1e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ec39ca1e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ec39ca1e
Branch: refs/heads/1.0.x-branch
Commit: ec39ca1e5f34ae0ab3e20abbba25d8bb52f9d650
Parents: 5d1e380 eb45904
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 7 10:39:02 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:39:02 2017 +0900
----------------------------------------------------------------------
.../org/apache/storm/daemon/supervisor/Container.java | 13 ++++++++++---
.../apache/storm/daemon/supervisor/ContainerTest.java | 6 +++---
2 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[09/13] storm git commit: Merge branch 'STORM-2335-1.0.x-merge' into
1.0.x-branch
Posted by ka...@apache.org.
Merge branch 'STORM-2335-1.0.x-merge' into 1.0.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/59ee79dd
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/59ee79dd
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/59ee79dd
Branch: refs/heads/1.0.x-branch
Commit: 59ee79ddf367010c7154e47f8ac135ade56cb1e6
Parents: ec39ca1 a22793c
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 7 10:40:08 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:40:08 2017 +0900
----------------------------------------------------------------------
storm-core/src/ui/public/js/visualization.js | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[11/13] storm git commit: Merge branch 'STORM-2336-1.0.x-merge' into
1.0.x-branch
Posted by ka...@apache.org.
Merge branch 'STORM-2336-1.0.x-merge' into 1.0.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a7b82c4b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a7b82c4b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a7b82c4b
Branch: refs/heads/1.0.x-branch
Commit: a7b82c4b66284ea0797d88827863d61ed50c854f
Parents: 59ee79d 64a9dd0
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 7 10:41:02 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:41:02 2017 +0900
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[05/13] storm git commit: Merge branch 'STORM-2321-1.0.x-merge' into
1.0.x-branch
Posted by ka...@apache.org.
Merge branch 'STORM-2321-1.0.x-merge' into 1.0.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5d1e3802
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5d1e3802
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5d1e3802
Branch: refs/heads/1.0.x-branch
Commit: 5d1e38023101ed45beea45e85fbba34582110f6a
Parents: 07ba34f 3b12349
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 7 10:37:45 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:37:45 2017 +0900
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/nimbus.clj | 6 +++++-
.../apache/storm/blobstore/BlobStoreUtils.java | 6 +++++-
.../storm/blobstore/BlobSynchronizer.java | 13 +++++++++---
.../storm/blobstore/KeySequenceNumber.java | 21 ++++++++++++++------
.../storm/blobstore/LocalFsBlobStore.java | 4 +++-
5 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[12/13] storm git commit: Merge branch
'fix/subrpocess_exception_handling' of https://github.com/tibkiss/storm into
STORM-2338-1.0.x-merge
Posted by ka...@apache.org.
Merge branch 'fix/subrpocess_exception_handling' of https://github.com/tibkiss/storm into STORM-2338-1.0.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/00b633d3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/00b633d3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/00b633d3
Branch: refs/heads/1.0.x-branch
Commit: 00b633d3b1fedd82df7559214c1d081415a8db63
Parents: a7b82c4 1b43804
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 7 10:41:43 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:41:43 2017 +0900
----------------------------------------------------------------------
bin/storm.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[06/13] storm git commit: STORM-2324 : Making sure toplogy deployment
does not break if resources directory is missing in topology jar. UT fixes
Posted by ka...@apache.org.
STORM-2324 : Making sure toplogy deployment does not break if resources directory is missing in topology jar. UT fixes
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/eb459042
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/eb459042
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/eb459042
Branch: refs/heads/1.0.x-branch
Commit: eb459042dc7c62e3b4de5d45a94f460ffd18031f
Parents: 5d1e380
Author: Roshan Naik <ro...@hortonworks.com>
Authored: Thu Feb 2 15:47:20 2017 -0800
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Feb 7 10:38:58 2017 +0900
----------------------------------------------------------------------
.../org/apache/storm/daemon/supervisor/Container.java | 13 ++++++++++---
.../apache/storm/daemon/supervisor/ContainerTest.java | 6 +++---
2 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/eb459042/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
index f4eb5e0..3f86b79 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
@@ -410,12 +410,19 @@ public abstract class Container implements Killable {
blobFileNames.add(ret);
}
}
+ File targetResourcesDir = new File(stormRoot, ConfigUtils.RESOURCES_SUBDIR);
List<String> resourceFileNames = new ArrayList<>();
- resourceFileNames.add(ConfigUtils.RESOURCES_SUBDIR);
+ if (targetResourcesDir.exists()) {
+ resourceFileNames.add(ConfigUtils.RESOURCES_SUBDIR);
+ }
resourceFileNames.addAll(blobFileNames);
+
LOG.info("Creating symlinks for worker-id: {} storm-id: {} for files({}): {}", _workerId, _topologyId, resourceFileNames.size(), resourceFileNames);
- _ops.createSymlink(new File(workerRoot, ConfigUtils.RESOURCES_SUBDIR),
- new File(stormRoot, ConfigUtils.RESOURCES_SUBDIR));
+ if(targetResourcesDir.exists()) {
+ _ops.createSymlink(new File(workerRoot, ConfigUtils.RESOURCES_SUBDIR), targetResourcesDir );
+ } else {
+ LOG.info("Topology jar for worker-id: {} storm-id: {} does not contain re sources directory {}." , _workerId, _topologyId, targetResourcesDir.toString() );
+ }
for (String fileName : blobFileNames) {
_ops.createSymlink(new File(workerRoot, fileName),
new File(stormRoot, fileName));
http://git-wip-us.apache.org/repos/asf/storm/blob/eb459042/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
index 3f34552..0958d4d 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
@@ -213,9 +213,9 @@ public class ContainerTest {
//Create links to artifacts dir
verify(ops).createSymlink(new File(workerRoot, "artifacts"), workerArtifacts);
-
- //Create links to blobs
- verify(ops).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources"));
+
+ //Create links to blobs
+ verify(ops, never()).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources"));
}
@Test