You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/12/20 08:56:38 UTC
[5/8] storm git commit: Added in some optimizations for better
topology submission performance
Added in some optimizations for better topology submission performance
Conflicts:
storm-core/src/jvm/org/apache/storm/StormSubmitter.java
storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/897a6d2b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/897a6d2b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/897a6d2b
Branch: refs/heads/1.x-branch
Commit: 897a6d2b9c2e34d06ba949c227d84f63904f5b5e
Parents: 93c1ea3
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Dec 2 15:28:17 2016 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Dec 5 07:48:44 2016 -0600
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/nimbus.clj | 28 +
.../jvm/org/apache/storm/StormSubmitter.java | 121 +-
.../storm/dependency/DependencyUploader.java | 22 +-
.../jvm/org/apache/storm/generated/Nimbus.java | 3377 ++++++++++++++----
.../org/apache/storm/utils/NimbusClient.java | 45 +-
.../StormBoundedExponentialBackoffRetry.java | 5 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 2 +
.../org/apache/storm/zookeeper/Zookeeper.java | 1 +
storm-core/src/py/storm/Nimbus-remote | 14 +
storm-core/src/py/storm/Nimbus.py | 385 ++
storm-core/src/storm.thrift | 2 +
11 files changed, 3137 insertions(+), 865 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/897a6d2b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
index e9d35a8..112d00b 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
@@ -94,6 +94,8 @@
(defmeter nimbus:num-getTopology-calls)
(defmeter nimbus:num-getUserTopology-calls)
(defmeter nimbus:num-getClusterInfo-calls)
+(defmeter nimbus:num-getLeader-calls)
+(defmeter nimbus:num-isTopologyNameAllowed-calls)
(defmeter nimbus:num-getTopologyInfoWithOpts-calls)
(defmeter nimbus:num-getTopologyInfo-calls)
(defmeter nimbus:num-getTopologyPageInfo-calls)
@@ -2353,6 +2355,32 @@
topo-history-list (read-topology-history nimbus user admin-users)]
(TopologyHistoryInfo. (distinct (concat active-ids-for-user topo-history-list)))))
+ (^NimbusSummary getLeader [this]
+ (mark! nimbus:num-getLeader-calls)
+ (check-authorization! nimbus nil nil "getClusterInfo")
+ (let [storm-cluster-state (:storm-cluster-state nimbus)
+ nimbuses (.nimbuses storm-cluster-state)
+ leader-elector (:leader-elector nimbus)
+ leader (.getLeader leader-elector)
+ leader-host (.getHost leader)
+ leader-port (.getPort leader)
+ leader-summary (first (filter
+ (fn [nimbus-summary] (and (= leader-host (.get_host nimbus-summary)) (= leader-port (.get_port nimbus-summary))))
+ nimbuses))]
+ (.set_uptime_secs leader-summary (time-delta (.get_uptime_secs leader-summary)))
+ (.set_isLeader leader-summary true)
+ leader-summary))
+
+ (^boolean isTopologyNameAllowed [this ^String name]
+ (mark! nimbus:num-isTopologyNameAllowed-calls)
+ (check-authorization! nimbus nil nil "getClusterInfo")
+ (try
+ (validate-topology-name! name)
+ (check-storm-active! nimbus name false)
+ true
+ (catch InvalidTopologyException e false)
+ (catch AlreadyAliveException e false)))
+
Shutdownable
(shutdown [this]
(mark! nimbus:num-shutdown-calls)
http://git-wip-us.apache.org/repos/asf/storm/blob/897a6d2b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
index effeed4..e414d60 100644
--- a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
@@ -128,19 +128,19 @@ public class StormSubmitter {
return;
}
try {
- if(localNimbus!=null) {
- LOG.info("Pushing Credentials to topology " + name + " in local mode");
+ if (localNimbus!=null) {
+ LOG.info("Pushing Credentials to topology {} in local mode", name);
localNimbus.uploadNewCredentials(name, new Credentials(fullCreds));
} else {
NimbusClient client = NimbusClient.getConfiguredClient(conf);
try {
- LOG.info("Uploading new credentials to " + name);
+ LOG.info("Uploading new credentials to {}", name);
client.getClient().uploadNewCredentials(name, new Credentials(fullCreds));
} finally {
client.close();
}
}
- LOG.info("Finished submitting topology: " + name);
+ LOG.info("Finished pushing creds to topology: {}", name);
} catch(TException e) {
throw new RuntimeException(e);
}
@@ -227,9 +227,9 @@ public class StormSubmitter {
opts.set_creds(new Credentials(fullCreds));
}
try {
- if(localNimbus!=null) {
+ if (localNimbus!=null) {
LOG.info("Submitting topology " + name + " in local mode");
- if(opts!=null) {
+ if (opts!=null) {
localNimbus.submitTopologyWithOpts(name, stormConf, topology, opts);
} else {
// this is for backwards compatibility
@@ -238,37 +238,41 @@ public class StormSubmitter {
LOG.info("Finished submitting topology: " + name);
} else {
String serConf = JSONValue.toJSONString(stormConf);
- if(topologyNameExists(conf, name, asUser)) {
- throw new RuntimeException("Topology with name `" + name + "` already exists on cluster");
- }
-
- // Dependency uploading only makes sense for distributed mode
- List<String> jarsBlobKeys = Collections.emptyList();
- List<String> artifactsBlobKeys;
-
- DependencyUploader uploader = new DependencyUploader();
- try {
- uploader.init();
- jarsBlobKeys = uploadDependencyJarsToBlobStore(uploader);
- artifactsBlobKeys = uploadDependencyArtifactsToBlobStore(uploader);
- } catch (Throwable e) {
- // remove uploaded jars blobs, not artifacts since they're shared across the cluster
- uploader.deleteBlobs(jarsBlobKeys);
- uploader.shutdown();
- throw e;
- }
-
- try {
- setDependencyBlobsToTopology(topology, jarsBlobKeys, artifactsBlobKeys);
- submitTopologyInDistributeMode(name, topology, opts, progressListener, asUser, conf, serConf);
- } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e) {
- // remove uploaded jars blobs, not artifacts since they're shared across the cluster
- // Note that we don't handle TException to delete jars blobs
- // because it's safer to leave some blobs instead of topology not running
- uploader.deleteBlobs(jarsBlobKeys);
- throw e;
- } finally {
- uploader.shutdown();
+ try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
+ if (topologyNameExists(name, client)) {
+ throw new RuntimeException("Topology with name `" + name + "` already exists on cluster");
+ }
+
+ // Dependency uploading only makes sense for distributed mode
+ List<String> jarsBlobKeys = Collections.emptyList();
+ List<String> artifactsBlobKeys;
+
+ DependencyUploader uploader = new DependencyUploader();
+ try {
+ uploader.init();
+
+ jarsBlobKeys = uploadDependencyJarsToBlobStore(uploader);
+
+ artifactsBlobKeys = uploadDependencyArtifactsToBlobStore(uploader);
+ } catch (Throwable e) {
+ // remove uploaded jars blobs, not artifacts since they're shared across the cluster
+ uploader.deleteBlobs(jarsBlobKeys);
+ uploader.shutdown();
+ throw e;
+ }
+
+ try {
+ setDependencyBlobsToTopology(topology, jarsBlobKeys, artifactsBlobKeys);
+ submitTopologyInDistributeMode(name, topology, opts, progressListener, asUser, conf, serConf, client);
+ } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e) {
+ // remove uploaded jars blobs, not artifacts since they're shared across the cluster
+ // Note that we don't handle TException to delete jars blobs
+ // because it's safer to leave some blobs instead of topology not running
+ uploader.deleteBlobs(jarsBlobKeys);
+ throw e;
+ } finally {
+ uploader.shutdown();
+ }
}
}
} catch(TException e) {
@@ -316,19 +320,20 @@ public class StormSubmitter {
private static void submitTopologyInDistributeMode(String name, StormTopology topology, SubmitOptions opts,
ProgressListener progressListener, String asUser, Map conf,
- String serConf) throws TException {
- String jar = submitJarAs(conf, System.getProperty("storm.jar"), progressListener, asUser);
- try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)){
- LOG.info("Submitting topology " + name + " in distributed mode with conf " + serConf);
+ String serConf, NimbusClient client) throws TException {
+ try {
+ String jar = submitJarAs(conf, System.getProperty("storm.jar"), progressListener, client);
+ LOG.info("Submitting topology {} in distributed mode with conf {}", name, serConf);
+
if (opts != null) {
client.getClient().submitTopologyWithOpts(name, jar, serConf, topology, opts);
} else {
// this is for backwards compatibility
client.getClient().submitTopology(name, jar, serConf, topology);
}
- LOG.info("Finished submitting topology: " + name);
+ LOG.info("Finished submitting topology: {}", name);
} catch (InvalidTopologyException e) {
- LOG.warn("Topology submission exception: " + e.get_msg());
+ LOG.warn("Topology submission exception: {}", e.get_msg());
throw e;
} catch (AlreadyAliveException e) {
LOG.warn("Topology already alive exception", e);
@@ -444,16 +449,9 @@ public class StormSubmitter {
});
}
- private static boolean topologyNameExists(Map conf, String name, String asUser) {
- try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
- ClusterSummary summary = client.getClient().getClusterInfo();
- for(TopologySummary s : summary.get_topologies()) {
- if(s.get_name().equals(name)) {
- return true;
- }
- }
- return false;
-
+ private static boolean topologyNameExists(String name, NimbusClient client) {
+ try {
+ return !client.getClient().isTopologyNameAllowed(name);
} catch(Exception e) {
throw new RuntimeException(e);
}
@@ -473,13 +471,12 @@ public class StormSubmitter {
return submitJar(conf, localJar, null);
}
-
- public static String submitJarAs(Map conf, String localJar, ProgressListener listener, String asUser) {
+ public static String submitJarAs(Map conf, String localJar, ProgressListener listener, NimbusClient client) {
if (localJar == null) {
throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
}
- try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
+ try {
String uploadLocation = client.getClient().beginFileUpload();
LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES);
@@ -512,6 +509,16 @@ public class StormSubmitter {
throw new RuntimeException(e);
}
}
+
+ public static String submitJarAs(Map conf, String localJar, ProgressListener listener, String asUser) {
+ if (localJar == null) {
+ throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
+ }
+
+ try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
+ return submitJarAs(conf, localJar, listener, client);
+ }
+ }
/**
* Submit jar file
@@ -521,7 +528,7 @@ public class StormSubmitter {
* @return the remote location of the submitted jar
*/
public static String submitJar(Map conf, String localJar, ProgressListener listener) {
- return submitJarAs(conf,localJar, listener, null);
+ return submitJarAs(conf,localJar, listener, (String)null);
}
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/897a6d2b/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java b/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
index 4f71f67..636f454 100644
--- a/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
+++ b/storm-core/src/jvm/org/apache/storm/dependency/DependencyUploader.java
@@ -18,7 +18,6 @@
package org.apache.storm.dependency;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
import org.apache.storm.blobstore.AtomicOutputStream;
import org.apache.storm.blobstore.BlobStoreUtils;
import org.apache.storm.blobstore.ClientBlobStore;
@@ -38,7 +37,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
public class DependencyUploader {
public static final Logger LOG = LoggerFactory.getLogger(DependencyUploader.class);
@@ -51,9 +49,7 @@ public class DependencyUploader {
}
public void init() {
- if (blobStore == null) {
- blobStore = Utils.getClientBlobStore(conf);
- }
+ //NOOP
}
public void shutdown() {
@@ -67,7 +63,13 @@ public class DependencyUploader {
this.blobStore = blobStore;
}
- @SuppressWarnings("unchecked")
+ private synchronized ClientBlobStore getBlobStore() {
+ if (blobStore == null) {
+ blobStore = Utils.getClientBlobStore(conf);
+ }
+ return blobStore;
+ }
+
public List<String> uploadFiles(List<File> dependencies, boolean cleanupIfFails) throws IOException, AuthorizationException {
checkFilesExist(dependencies);
@@ -87,7 +89,7 @@ public class DependencyUploader {
keys.add(key);
}
} catch (Throwable e) {
- if (blobStore != null && cleanupIfFails) {
+ if (getBlobStore() != null && cleanupIfFails) {
deleteBlobs(keys);
}
throw new RuntimeException(e);
@@ -124,7 +126,7 @@ public class DependencyUploader {
public void deleteBlobs(List<String> keys) {
for (String key : keys) {
try {
- blobStore.deleteBlob(key);
+ getBlobStore().deleteBlob(key);
} catch (Throwable e) {
LOG.warn("blob delete failed - key: {} continue...", key);
}
@@ -142,10 +144,10 @@ public class DependencyUploader {
try {
// FIXME: we can filter by listKeys() with local blobstore when STORM-1986 is going to be resolved
// as a workaround, we call getBlobMeta() for all keys
- blobStore.getBlobMeta(key);
+ getBlobStore().getBlobMeta(key);
} catch (KeyNotFoundException e) {
// TODO: do we want to add ACL here?
- AtomicOutputStream blob = blobStore
+ AtomicOutputStream blob = getBlobStore()
.createBlob(key, new SettableBlobMeta(new ArrayList<AccessControl>()));
Files.copy(dependency.toPath(), blob);
blob.close();