You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/12/20 08:56:38 UTC

[5/8] storm git commit: Added in some optimizations for better topology submission performance

Added in some optimizations for better topology submission performance

Conflicts:
	storm-core/src/jvm/org/apache/storm/StormSubmitter.java
	storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/897a6d2b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/897a6d2b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/897a6d2b

Branch: refs/heads/1.x-branch
Commit: 897a6d2b9c2e34d06ba949c227d84f63904f5b5e
Parents: 93c1ea3
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Dec 2 15:28:17 2016 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Dec 5 07:48:44 2016 -0600

----------------------------------------------------------------------
 .../src/clj/org/apache/storm/daemon/nimbus.clj  |   28 +
 .../jvm/org/apache/storm/StormSubmitter.java    |  121 +-
 .../storm/dependency/DependencyUploader.java    |   22 +-
 .../jvm/org/apache/storm/generated/Nimbus.java  | 3377 ++++++++++++++----
 .../org/apache/storm/utils/NimbusClient.java    |   45 +-
 .../StormBoundedExponentialBackoffRetry.java    |    5 +-
 .../src/jvm/org/apache/storm/utils/Utils.java   |    2 +
 .../org/apache/storm/zookeeper/Zookeeper.java   |    1 +
 storm-core/src/py/storm/Nimbus-remote           |   14 +
 storm-core/src/py/storm/Nimbus.py               |  385 ++
 storm-core/src/storm.thrift                     |    2 +
 11 files changed, 3137 insertions(+), 865 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/897a6d2b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
index e9d35a8..112d00b 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
@@ -94,6 +94,8 @@
 (defmeter nimbus:num-getTopology-calls)
 (defmeter nimbus:num-getUserTopology-calls)
 (defmeter nimbus:num-getClusterInfo-calls)
+(defmeter nimbus:num-getLeader-calls)
+(defmeter nimbus:num-isTopologyNameAllowed-calls)
 (defmeter nimbus:num-getTopologyInfoWithOpts-calls)
 (defmeter nimbus:num-getTopologyInfo-calls)
 (defmeter nimbus:num-getTopologyPageInfo-calls)
@@ -2353,6 +2355,32 @@
             topo-history-list (read-topology-history nimbus user admin-users)]
         (TopologyHistoryInfo. (distinct (concat active-ids-for-user topo-history-list)))))
 
+    (^NimbusSummary getLeader [this]
+      (mark! nimbus:num-getLeader-calls)
+      (check-authorization! nimbus nil nil "getClusterInfo")
+      (let [storm-cluster-state (:storm-cluster-state nimbus)
+            nimbuses (.nimbuses storm-cluster-state)
+            leader-elector (:leader-elector nimbus)
+            leader (.getLeader leader-elector)
+            leader-host (.getHost leader)
+            leader-port (.getPort leader)
+            leader-summary (first (filter
+               (fn [nimbus-summary] (and (= leader-host (.get_host nimbus-summary)) (= leader-port (.get_port nimbus-summary))))
+               nimbuses))]
+        (.set_uptime_secs leader-summary (time-delta (.get_uptime_secs leader-summary)))
+        (.set_isLeader leader-summary true)
+        leader-summary))
+        
+    (^boolean isTopologyNameAllowed [this ^String name]
+      (mark! nimbus:num-isTopologyNameAllowed-calls)
+      (check-authorization! nimbus nil nil "getClusterInfo")
+      (try
+        (validate-topology-name! name)
+        (check-storm-active! nimbus name false)
+        true
+        (catch InvalidTopologyException e false)
+        (catch AlreadyAliveException e false)))
+ 
     Shutdownable
     (shutdown [this]
       (mark! nimbus:num-shutdown-calls)

http://git-wip-us.apache.org/repos/asf/storm/blob/897a6d2b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
index effeed4..e414d60 100644
--- a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
@@ -128,19 +128,19 @@ public class StormSubmitter {
             return;
         }
         try {
-            if(localNimbus!=null) {
-                LOG.info("Pushing Credentials to topology " + name + " in local mode");
+            if (localNimbus!=null) {
+                LOG.info("Pushing Credentials to topology {} in local mode", name);
                 localNimbus.uploadNewCredentials(name, new Credentials(fullCreds));
             } else {
                 NimbusClient client = NimbusClient.getConfiguredClient(conf);
                 try {
-                    LOG.info("Uploading new credentials to " +  name);
+                    LOG.info("Uploading new credentials to {}", name);
                     client.getClient().uploadNewCredentials(name, new Credentials(fullCreds));
                 } finally {
                     client.close();
                 }
             }
-            LOG.info("Finished submitting topology: " +  name);
+            LOG.info("Finished pushing creds to topology: {}", name);
         } catch(TException e) {
             throw new RuntimeException(e);
         }
@@ -227,9 +227,9 @@ public class StormSubmitter {
             opts.set_creds(new Credentials(fullCreds));
         }
         try {
-            if(localNimbus!=null) {
+            if (localNimbus!=null) {
                 LOG.info("Submitting topology " + name + " in local mode");
-                if(opts!=null) {
+                if (opts!=null) {
                     localNimbus.submitTopologyWithOpts(name, stormConf, topology, opts);
                 } else {
                     // this is for backwards compatibility
@@ -238,37 +238,41 @@ public class StormSubmitter {
                 LOG.info("Finished submitting topology: " +  name);
             } else {
                 String serConf = JSONValue.toJSONString(stormConf);
-                if(topologyNameExists(conf, name, asUser)) {
-                    throw new RuntimeException("Topology with name `" + name + "` already exists on cluster");
-                }
-
-                // Dependency uploading only makes sense for distributed mode
-                List<String> jarsBlobKeys = Collections.emptyList();
-                List<String> artifactsBlobKeys;
-
-                DependencyUploader uploader = new DependencyUploader();
-                try {
-                    uploader.init();
-                    jarsBlobKeys = uploadDependencyJarsToBlobStore(uploader);
-                    artifactsBlobKeys = uploadDependencyArtifactsToBlobStore(uploader);
-                } catch (Throwable e) {
-                    // remove uploaded jars blobs, not artifacts since they're shared across the cluster
-                    uploader.deleteBlobs(jarsBlobKeys);
-                    uploader.shutdown();
-                    throw e;
-                }
-
-                try {
-                    setDependencyBlobsToTopology(topology, jarsBlobKeys, artifactsBlobKeys);
-                    submitTopologyInDistributeMode(name, topology, opts, progressListener, asUser, conf, serConf);
-                } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e) {
-                    // remove uploaded jars blobs, not artifacts since they're shared across the cluster
-                    // Note that we don't handle TException to delete jars blobs
-                    // because it's safer to leave some blobs instead of topology not running
-                    uploader.deleteBlobs(jarsBlobKeys);
-                    throw e;
-                } finally {
-                    uploader.shutdown();
+                try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
+                    if (topologyNameExists(name, client)) {
+                        throw new RuntimeException("Topology with name `" + name + "` already exists on cluster");
+                    }
+
+                    // Dependency uploading only makes sense for distributed mode
+                    List<String> jarsBlobKeys = Collections.emptyList();
+                    List<String> artifactsBlobKeys;
+
+                    DependencyUploader uploader = new DependencyUploader();
+                    try {
+                        uploader.init();
+
+                        jarsBlobKeys = uploadDependencyJarsToBlobStore(uploader);
+
+                        artifactsBlobKeys = uploadDependencyArtifactsToBlobStore(uploader);
+                    } catch (Throwable e) {
+                        // remove uploaded jars blobs, not artifacts since they're shared across the cluster
+                        uploader.deleteBlobs(jarsBlobKeys);
+                        uploader.shutdown();
+                        throw e;
+                    }
+
+                    try {
+                        setDependencyBlobsToTopology(topology, jarsBlobKeys, artifactsBlobKeys);
+                        submitTopologyInDistributeMode(name, topology, opts, progressListener, asUser, conf, serConf, client);
+                    } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e) {
+                        // remove uploaded jars blobs, not artifacts since they're shared across the cluster
+                        // Note that we don't handle TException to delete jars blobs
+                        // because it's safer to leave some blobs instead of topology not running
+                        uploader.deleteBlobs(jarsBlobKeys);
+                        throw e;
+                    } finally {
+                        uploader.shutdown();
+                    }
                 }
             }
         } catch(TException e) {
@@ -316,19 +320,20 @@ public class StormSubmitter {
 
     private static void submitTopologyInDistributeMode(String name, StormTopology topology, SubmitOptions opts,
                                                        ProgressListener progressListener, String asUser, Map conf,
-                                                       String serConf) throws TException {
-        String jar = submitJarAs(conf, System.getProperty("storm.jar"), progressListener, asUser);
-        try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)){
-            LOG.info("Submitting topology " + name + " in distributed mode with conf " + serConf);
+                                                       String serConf, NimbusClient client) throws TException {
+        try {
+            String jar = submitJarAs(conf, System.getProperty("storm.jar"), progressListener, client);
+            LOG.info("Submitting topology {} in distributed mode with conf {}", name, serConf);
+
             if (opts != null) {
                 client.getClient().submitTopologyWithOpts(name, jar, serConf, topology, opts);
             } else {
                 // this is for backwards compatibility
                 client.getClient().submitTopology(name, jar, serConf, topology);
             }
-            LOG.info("Finished submitting topology: " + name);
+            LOG.info("Finished submitting topology: {}", name);
         } catch (InvalidTopologyException e) {
-            LOG.warn("Topology submission exception: " + e.get_msg());
+            LOG.warn("Topology submission exception: {}", e.get_msg());
             throw e;
         } catch (AlreadyAliveException e) {
             LOG.warn("Topology already alive exception", e);
@@ -444,16 +449,9 @@ public class StormSubmitter {
         });
     }
 
-    private static boolean topologyNameExists(Map conf, String name, String asUser) {
-        try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
-            ClusterSummary summary = client.getClient().getClusterInfo();
-            for(TopologySummary s : summary.get_topologies()) {
-                if(s.get_name().equals(name)) {
-                    return true;
-                }
-            }
-            return false;
-
+    private static boolean topologyNameExists(String name, NimbusClient client) {
+        try {
+            return !client.getClient().isTopologyNameAllowed(name);
         } catch(Exception e) {
             throw new RuntimeException(e);
         }
@@ -473,13 +471,12 @@ public class StormSubmitter {
         return submitJar(conf, localJar, null);
     }
 
-
-    public static String submitJarAs(Map conf, String localJar, ProgressListener listener, String asUser) {
+    public static String submitJarAs(Map conf, String localJar, ProgressListener listener, NimbusClient client) {
         if (localJar == null) {
             throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
         }
 
-        try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
+        try {
             String uploadLocation = client.getClient().beginFileUpload();
             LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
             BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES);
@@ -512,6 +509,16 @@ public class StormSubmitter {
             throw new RuntimeException(e);
         }
     }
+    
+    public static String submitJarAs(Map conf, String localJar, ProgressListener listener, String asUser) {
+        if (localJar == null) {
+            throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
+        }
+
+        try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
+            return submitJarAs(conf, localJar, listener, client);
+        }
+    }
 
     /**
      * Submit jar file
@@ -521,7 +528,7 @@ public class StormSubmitter {
      * @return the remote location of the submitted jar
      */
     public static String submitJar(Map conf, String localJar, ProgressListener listener) {
-        return submitJarAs(conf,localJar, listener, null);
+        return submitJarAs(conf,localJar, listener, (String)null);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/storm/blob/897a6d2b/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java b/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
index 4f71f67..636f454 100644
--- a/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
+++ b/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
@@ -18,7 +18,6 @@
 package org.apache.storm.dependency;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
 import org.apache.storm.blobstore.AtomicOutputStream;
 import org.apache.storm.blobstore.BlobStoreUtils;
 import org.apache.storm.blobstore.ClientBlobStore;
@@ -38,7 +37,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 
 public class DependencyUploader {
     public static final Logger LOG = LoggerFactory.getLogger(DependencyUploader.class);
@@ -51,9 +49,7 @@ public class DependencyUploader {
     }
 
     public void init() {
-        if (blobStore == null) {
-            blobStore = Utils.getClientBlobStore(conf);
-        }
+        //NOOP
     }
 
     public void shutdown() {
@@ -67,7 +63,13 @@ public class DependencyUploader {
         this.blobStore = blobStore;
     }
 
-    @SuppressWarnings("unchecked")
+    private synchronized ClientBlobStore getBlobStore() {
+        if (blobStore == null) {
+            blobStore = Utils.getClientBlobStore(conf);
+        }
+        return blobStore;
+    }
+
     public List<String> uploadFiles(List<File> dependencies, boolean cleanupIfFails) throws IOException, AuthorizationException {
         checkFilesExist(dependencies);
 
@@ -87,7 +89,7 @@ public class DependencyUploader {
                 keys.add(key);
             }
         } catch (Throwable e) {
-            if (blobStore != null && cleanupIfFails) {
+            if (getBlobStore() != null && cleanupIfFails) {
                 deleteBlobs(keys);
             }
             throw new RuntimeException(e);
@@ -124,7 +126,7 @@ public class DependencyUploader {
     public void deleteBlobs(List<String> keys) {
         for (String key : keys) {
             try {
-                blobStore.deleteBlob(key);
+                getBlobStore().deleteBlob(key);
             } catch (Throwable e) {
                 LOG.warn("blob delete failed - key: {} continue...", key);
             }
@@ -142,10 +144,10 @@ public class DependencyUploader {
         try {
             // FIXME: we can filter by listKeys() with local blobstore when STORM-1986 is going to be resolved
             // as a workaround, we call getBlobMeta() for all keys
-            blobStore.getBlobMeta(key);
+            getBlobStore().getBlobMeta(key);
         } catch (KeyNotFoundException e) {
             // TODO: do we want to add ACL here?
-            AtomicOutputStream blob = blobStore
+            AtomicOutputStream blob = getBlobStore()
                     .createBlob(key, new SettableBlobMeta(new ArrayList<AccessControl>()));
             Files.copy(dependency.toPath(), blob);
             blob.close();