You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/01/03 14:31:45 UTC

[1/4] storm git commit: STORM-2266 Close NimbusClient instances appropriately

Repository: storm
Updated Branches:
  refs/heads/master 46121cbd4 -> f14517493


STORM-2266 Close NimbusClient instances appropriately


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5ab547e8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5ab547e8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5ab547e8

Branch: refs/heads/master
Commit: 5ab547e82b724550e577f795845d9fa16475ab05
Parents: 02ddf88
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Thu Dec 29 12:54:43 2016 +0530
Committer: Satish Duggana <sd...@hortonworks.com>
Committed: Thu Dec 29 13:32:47 2016 +0530

----------------------------------------------------------------------
 .../apache/storm/hdfs/blobstore/HdfsClientBlobStore.java    | 7 ++++++-
 storm-core/src/jvm/org/apache/storm/StormSubmitter.java     | 5 +----
 .../src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java  | 8 ++------
 storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java | 9 +--------
 4 files changed, 10 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
index 1fc80d0..d88211c 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
@@ -45,6 +45,7 @@ public class HdfsClientBlobStore extends ClientBlobStore {
     private static final Logger LOG = LoggerFactory.getLogger(HdfsClientBlobStore.class);
     private HdfsBlobStore _blobStore;
     private Map _conf;
+    private NimbusClient client;
 
     @Override
     public void prepare(Map conf) {
@@ -105,6 +106,7 @@ public class HdfsClientBlobStore extends ClientBlobStore {
 
     @Override
     public boolean setClient(Map conf, NimbusClient client) {
+        this.client = client;
         return true;
     }
 
@@ -115,6 +117,9 @@ public class HdfsClientBlobStore extends ClientBlobStore {
 
     @Override
     public void shutdown() {
-        // do nothing
+        if(client != null) {
+            client.close();
+            client = null;
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
index e414d60..d382dd9 100644
--- a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
@@ -132,12 +132,9 @@ public class StormSubmitter {
                 LOG.info("Pushing Credentials to topology {} in local mode", name);
                 localNimbus.uploadNewCredentials(name, new Credentials(fullCreds));
             } else {
-                NimbusClient client = NimbusClient.getConfiguredClient(conf);
-                try {
+                try(NimbusClient client = NimbusClient.getConfiguredClient(conf)) {
                     LOG.info("Uploading new credentials to {}", name);
                     client.getClient().uploadNewCredentials(name, new Credentials(fullCreds));
-                } finally {
-                    client.close();
                 }
             }
             LOG.info("Finished pushing creds to topology: {}", name);

http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
index c550cfe..ba3ccdb 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
@@ -110,7 +110,6 @@ public class BlobStoreUtils {
     // Download missing blobs from potential nimbodes
     public static boolean downloadMissingBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
             throws TTransportException {
-        NimbusClient client;
         ReadableBlobMeta rbm;
         ClientBlobStore remoteBlobStore;
         InputStreamWithMeta in;
@@ -120,8 +119,7 @@ public class BlobStoreUtils {
             if(isSuccess) {
                 break;
             }
-            try {
-                client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null);
+            try(NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
                 rbm = client.getClient().getBlobMeta(key);
                 remoteBlobStore = new NimbusBlobStore();
                 remoteBlobStore.setClient(conf, client);
@@ -160,7 +158,6 @@ public class BlobStoreUtils {
     // Download updated blobs from potential nimbodes
     public static boolean downloadUpdatedBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
             throws TTransportException {
-        NimbusClient client;
         ClientBlobStore remoteBlobStore;
         InputStreamWithMeta in;
         AtomicOutputStream out;
@@ -170,8 +167,7 @@ public class BlobStoreUtils {
             if (isSuccess) {
                 break;
             }
-            try {
-                client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null);
+            try(NimbusClient client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)) {
                 remoteBlobStore = new NimbusBlobStore();
                 remoteBlobStore.setClient(conf, client);
                 in = remoteBlobStore.getBlob(key);

http://git-wip-us.apache.org/repos/asf/storm/blob/5ab547e8/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
index 85209d6..aaddf3d 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -78,16 +78,13 @@ public class NimbusClient extends ThriftClient {
         for (String host : seeds) {
             int port = Integer.parseInt(conf.get(Config.NIMBUS_THRIFT_PORT).toString());
             NimbusSummary nimbusSummary;
-            NimbusClient client = null;
-            try {
-                client = new NimbusClient(conf, host, port, null, asUser);
+            try(NimbusClient client = new NimbusClient(conf, host, port, null, asUser)) {
                 nimbusSummary = client.getClient().getLeader();
                 if (nimbusSummary != null) {
                     String leaderNimbus = nimbusSummary.get_host() + ":" + nimbusSummary.get_port();
                     LOG.info("Found leader nimbus : {}", leaderNimbus);
                     if (nimbusSummary.get_host().equals(host) && nimbusSummary.get_port() == port) {
                         NimbusClient ret = client;
-                        client = null;
                         return ret;
                     }
                     try {
@@ -100,10 +97,6 @@ public class NimbusClient extends ThriftClient {
                 LOG.warn("Ignoring exception while trying to get leader nimbus info from " + host
                         + ". will retry with a different seed host.", e);
                 continue;
-            } finally {
-                if (client != null) {
-                    client.close();
-                }
             }
             throw new NimbusLeaderNotFoundException("Could not find a nimbus leader, please try " +
                     "again after some time.");


[3/4] storm git commit: Merge branch 'STORM-2266-m' of https://github.com/satishd/storm into STORM-2266-merge

Posted by ka...@apache.org.
Merge branch 'STORM-2266-m' of https://github.com/satishd/storm into STORM-2266-merge


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/add21b67
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/add21b67
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/add21b67

Branch: refs/heads/master
Commit: add21b67763ea3f295028a68a5536d28c493ec42
Parents: 46121cb fe013c4
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Jan 3 23:23:43 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Jan 3 23:23:43 2017 +0900

----------------------------------------------------------------------
 .../org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java | 7 ++++++-
 storm-core/src/jvm/org/apache/storm/StormSubmitter.java      | 5 +----
 .../src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java   | 8 ++------
 storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java  | 8 ++++----
 4 files changed, 13 insertions(+), 15 deletions(-)
----------------------------------------------------------------------



[2/4] storm git commit: STORM-2266: Addressed review comments

Posted by ka...@apache.org.
STORM-2266: Addressed review comments


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fe013c45
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fe013c45
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fe013c45

Branch: refs/heads/master
Commit: fe013c45d869107834b6d0e188e5019fdbc09351
Parents: 5ab547e
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Tue Jan 3 07:01:51 2017 +0530
Committer: Satish Duggana <sd...@hortonworks.com>
Committed: Tue Jan 3 07:03:05 2017 +0530

----------------------------------------------------------------------
 .../jvm/org/apache/storm/utils/NimbusClient.java   | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/fe013c45/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
index aaddf3d..6699ba8 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -69,7 +69,7 @@ public class NimbusClient extends ThriftClient {
         List<String> seeds;
         if(conf.containsKey(Config.NIMBUS_HOST)) {
             LOG.warn("Using deprecated config {} for backward compatibility. Please update your storm.yaml so it only has config {}",
-                    Config.NIMBUS_HOST, Config.NIMBUS_SEEDS);
+                     Config.NIMBUS_HOST, Config.NIMBUS_SEEDS);
             seeds = Lists.newArrayList(conf.get(Config.NIMBUS_HOST).toString());
         } else {
             seeds = (List<String>) conf.get(Config.NIMBUS_SEEDS);
@@ -78,13 +78,16 @@ public class NimbusClient extends ThriftClient {
         for (String host : seeds) {
             int port = Integer.parseInt(conf.get(Config.NIMBUS_THRIFT_PORT).toString());
             NimbusSummary nimbusSummary;
-            try(NimbusClient client = new NimbusClient(conf, host, port, null, asUser)) {
+            NimbusClient client = null;
+            try {
+                client = new NimbusClient(conf, host, port, null, asUser);
                 nimbusSummary = client.getClient().getLeader();
                 if (nimbusSummary != null) {
                     String leaderNimbus = nimbusSummary.get_host() + ":" + nimbusSummary.get_port();
                     LOG.info("Found leader nimbus : {}", leaderNimbus);
                     if (nimbusSummary.get_host().equals(host) && nimbusSummary.get_port() == port) {
                         NimbusClient ret = client;
+                        client = null;
                         return ret;
                     }
                     try {
@@ -95,15 +98,19 @@ public class NimbusClient extends ThriftClient {
                 }
             } catch (Exception e) {
                 LOG.warn("Ignoring exception while trying to get leader nimbus info from " + host
-                        + ". will retry with a different seed host.", e);
+                                 + ". will retry with a different seed host.", e);
                 continue;
+            } finally {
+                if (client != null) {
+                    client.close();
+                }
             }
             throw new NimbusLeaderNotFoundException("Could not find a nimbus leader, please try " +
-                    "again after some time.");
+                                                            "again after some time.");
         }
         throw new NimbusLeaderNotFoundException(
                 "Could not find leader nimbus from seed hosts " + seeds + ". " +
-                "Did you specify a valid list of nimbus hosts for config " +
+                        "Did you specify a valid list of nimbus hosts for config " +
                         Config.NIMBUS_SEEDS + "?");
     }
 


[4/4] storm git commit: STORM-2266: CHANGELOG

Posted by ka...@apache.org.
STORM-2266: CHANGELOG


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f1451749
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f1451749
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f1451749

Branch: refs/heads/master
Commit: f14517493d6452d41c94ddf3d6c0bb2f630e3b03
Parents: add21b6
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Jan 3 23:31:15 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Jan 3 23:31:15 2017 +0900

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/f1451749/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6acadb0..bed79b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -188,6 +188,7 @@
  * STORM-1769: Added a test to check local nimbus with notifier plugin
 
 ## 1.1.0
+ * STORM-2266: Close NimbusClient instances appropriately
  * STORM-2257: Add built in support for sum function with different types
  * STORM-2082: add sql external module storm-sql-hdfs
  * STORM-2190: reduce contention between submission and scheduling