You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/01/03 14:31:45 UTC
[1/4] storm git commit: STORM-2266 Close NimbusClient instances
appropriately
Repository: storm
Updated Branches:
refs/heads/master 46121cbd4 -> f14517493
STORM-2266 Close NimbusClient instances appropriately
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5ab547e8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5ab547e8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5ab547e8
Branch: refs/heads/master
Commit: 5ab547e82b724550e577f795845d9fa16475ab05
Parents: 02ddf88
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Thu Dec 29 12:54:43 2016 +0530
Committer: Satish Duggana <sd...@hortonworks.com>
Committed: Thu Dec 29 13:32:47 2016 +0530
----------------------------------------------------------------------
.../apache/storm/hdfs/blobstore/HdfsClientBlobStore.java | 7 ++++++-
storm-core/src/jvm/org/apache/storm/StormSubmitter.java | 5 +----
.../src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java | 8 ++------
storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java | 9 +--------
4 files changed, 10 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
index 1fc80d0..d88211c 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
@@ -45,6 +45,7 @@ public class HdfsClientBlobStore extends ClientBlobStore {
private static final Logger LOG = LoggerFactory.getLogger(HdfsClientBlobStore.class);
private HdfsBlobStore _blobStore;
private Map _conf;
+ private NimbusClient client;
@Override
public void prepare(Map conf) {
@@ -105,6 +106,7 @@ public class HdfsClientBlobStore extends ClientBlobStore {
@Override
public boolean setClient(Map conf, NimbusClient client) {
+ this.client = client;
return true;
}
@@ -115,6 +117,9 @@ public class HdfsClientBlobStore extends ClientBlobStore {
@Override
public void shutdown() {
- // do nothing
+ if(client != null) {
+ client.close();
+ client = null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
index e414d60..d382dd9 100644
--- a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
@@ -132,12 +132,9 @@ public class StormSubmitter {
LOG.info("Pushing Credentials to topology {} in local mode", name);
localNimbus.uploadNewCredentials(name, new Credentials(fullCreds));
} else {
- NimbusClient client = NimbusClient.getConfiguredClient(conf);
- try {
+ try(NimbusClient client = NimbusClient.getConfiguredClient(conf)) {
LOG.info("Uploading new credentials to {}", name);
client.getClient().uploadNewCredentials(name, new Credentials(fullCreds));
- } finally {
- client.close();
}
}
LOG.info("Finished pushing creds to topology: {}", name);
http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
index c550cfe..ba3ccdb 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
@@ -110,7 +110,6 @@ public class BlobStoreUtils {
// Download missing blobs from potential nimbodes
public static boolean downloadMissingBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
throws TTransportException {
- NimbusClient client;
ReadableBlobMeta rbm;
ClientBlobStore remoteBlobStore;
InputStreamWithMeta in;
@@ -120,8 +119,7 @@ public class BlobStoreUtils {
if(isSuccess) {
break;
}
- try {
- client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null);
+ try(NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
rbm = client.getClient().getBlobMeta(key);
remoteBlobStore = new NimbusBlobStore();
remoteBlobStore.setClient(conf, client);
@@ -160,7 +158,6 @@ public class BlobStoreUtils {
// Download updated blobs from potential nimbodes
public static boolean downloadUpdatedBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
throws TTransportException {
- NimbusClient client;
ClientBlobStore remoteBlobStore;
InputStreamWithMeta in;
AtomicOutputStream out;
@@ -170,8 +167,7 @@ public class BlobStoreUtils {
if (isSuccess) {
break;
}
- try {
- client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null);
+ try(NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
remoteBlobStore = new NimbusBlobStore();
remoteBlobStore.setClient(conf, client);
in = remoteBlobStore.getBlob(key);
http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
index 85209d6..aaddf3d 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -78,16 +78,13 @@ public class NimbusClient extends ThriftClient {
for (String host : seeds) {
int port = Integer.parseInt(conf.get(Config.NIMBUS_THRIFT_PORT).toString());
NimbusSummary nimbusSummary;
- NimbusClient client = null;
- try {
- client = new NimbusClient(conf, host, port, null, asUser);
+ try(NimbusClient client = new NimbusClient(conf, host, port, null, asUser)) {
nimbusSummary = client.getClient().getLeader();
if (nimbusSummary != null) {
String leaderNimbus = nimbusSummary.get_host() + ":" + nimbusSummary.get_port();
LOG.info("Found leader nimbus : {}", leaderNimbus);
if (nimbusSummary.get_host().equals(host) && nimbusSummary.get_port() == port) {
NimbusClient ret = client;
- client = null;
return ret;
}
try {
@@ -100,10 +97,6 @@ public class NimbusClient extends ThriftClient {
LOG.warn("Ignoring exception while trying to get leader nimbus info from " + host
+ ". will retry with a different seed host.", e);
continue;
- } finally {
- if (client != null) {
- client.close();
- }
}
throw new NimbusLeaderNotFoundException("Could not find a nimbus leader, please try " +
"again after some time.");
[3/4] storm git commit: Merge branch 'STORM-2266-m' of
https://github.com/satishd/storm into STORM-2266-merge
Posted by ka...@apache.org.
Merge branch 'STORM-2266-m' of https://github.com/satishd/storm into STORM-2266-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/add21b67
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/add21b67
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/add21b67
Branch: refs/heads/master
Commit: add21b67763ea3f295028a68a5536d28c493ec42
Parents: 46121cb fe013c4
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Jan 3 23:23:43 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Jan 3 23:23:43 2017 +0900
----------------------------------------------------------------------
.../org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java | 7 ++++++-
storm-core/src/jvm/org/apache/storm/StormSubmitter.java | 5 +----
.../src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java | 8 ++------
storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java | 8 ++++----
4 files changed, 13 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
[2/4] storm git commit: STORM-2266: Addressed review comments
Posted by ka...@apache.org.
STORM-2266: Addressed review comments
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fe013c45
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fe013c45
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fe013c45
Branch: refs/heads/master
Commit: fe013c45d869107834b6d0e188e5019fdbc09351
Parents: 5ab547e
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Tue Jan 3 07:01:51 2017 +0530
Committer: Satish Duggana <sd...@hortonworks.com>
Committed: Tue Jan 3 07:03:05 2017 +0530
----------------------------------------------------------------------
.../jvm/org/apache/storm/utils/NimbusClient.java | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/fe013c45/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
index aaddf3d..6699ba8 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -69,7 +69,7 @@ public class NimbusClient extends ThriftClient {
List<String> seeds;
if(conf.containsKey(Config.NIMBUS_HOST)) {
LOG.warn("Using deprecated config {} for backward compatibility. Please update your storm.yaml so it only has config {}",
- Config.NIMBUS_HOST, Config.NIMBUS_SEEDS);
+ Config.NIMBUS_HOST, Config.NIMBUS_SEEDS);
seeds = Lists.newArrayList(conf.get(Config.NIMBUS_HOST).toString());
} else {
seeds = (List<String>) conf.get(Config.NIMBUS_SEEDS);
@@ -78,13 +78,16 @@ public class NimbusClient extends ThriftClient {
for (String host : seeds) {
int port = Integer.parseInt(conf.get(Config.NIMBUS_THRIFT_PORT).toString());
NimbusSummary nimbusSummary;
- try(NimbusClient client = new NimbusClient(conf, host, port, null, asUser)) {
+ NimbusClient client = null;
+ try {
+ client = new NimbusClient(conf, host, port, null, asUser);
nimbusSummary = client.getClient().getLeader();
if (nimbusSummary != null) {
String leaderNimbus = nimbusSummary.get_host() + ":" + nimbusSummary.get_port();
LOG.info("Found leader nimbus : {}", leaderNimbus);
if (nimbusSummary.get_host().equals(host) && nimbusSummary.get_port() == port) {
NimbusClient ret = client;
+ client = null;
return ret;
}
try {
@@ -95,15 +98,19 @@ public class NimbusClient extends ThriftClient {
}
} catch (Exception e) {
LOG.warn("Ignoring exception while trying to get leader nimbus info from " + host
- + ". will retry with a different seed host.", e);
+ + ". will retry with a different seed host.", e);
continue;
+ } finally {
+ if (client != null) {
+ client.close();
+ }
}
throw new NimbusLeaderNotFoundException("Could not find a nimbus leader, please try " +
- "again after some time.");
+ "again after some time.");
}
throw new NimbusLeaderNotFoundException(
"Could not find leader nimbus from seed hosts " + seeds + ". " +
- "Did you specify a valid list of nimbus hosts for config " +
+ "Did you specify a valid list of nimbus hosts for config " +
Config.NIMBUS_SEEDS + "?");
}
[4/4] storm git commit: STORM-2266: CHANGELOG
Posted by ka...@apache.org.
STORM-2266: CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f1451749
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f1451749
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f1451749
Branch: refs/heads/master
Commit: f14517493d6452d41c94ddf3d6c0bb2f630e3b03
Parents: add21b6
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Jan 3 23:31:15 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Jan 3 23:31:15 2017 +0900
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f1451749/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6acadb0..bed79b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -188,6 +188,7 @@
* STORM-1769: Added a test to check local nimbus with notifier plugin
## 1.1.0
+ * STORM-2266: Close NimbusClient instances appropriately
* STORM-2257: Add built in support for sum function with different types
* STORM-2082: add sql external module storm-sql-hdfs
* STORM-2190: reduce contention between submission and scheduling