You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2015/11/05 21:40:49 UTC
[10/60] [abbrv] [partial] storm git commit: Release 2.0.4-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/Constants.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/Constants.java b/jstorm-core/src/main/java/backtype/storm/Constants.java
new file mode 100755
index 0000000..2797b69
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/Constants.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import backtype.storm.coordination.CoordinatedBolt;
+import clojure.lang.RT;
+
+
+public class Constants {
+ public static final String COORDINATED_STREAM_ID = CoordinatedBolt.class.getName() + "/coord-stream";
+
+ public static final long SYSTEM_TASK_ID = -1;
+ public static final Object SYSTEM_EXECUTOR_ID = RT.readString("[-1 -1]");
+ public static final String SYSTEM_COMPONENT_ID = "__system";
+ public static final String SYSTEM_TICK_STREAM_ID = "__tick";
+ public static final String METRICS_COMPONENT_ID_PREFIX = "__metrics";
+ public static final String METRICS_STREAM_ID = "__metrics";
+ public static final String METRICS_TICK_STREAM_ID = "__metrics_tick";
+ public static final String CREDENTIALS_CHANGED_STREAM_ID = "__credentials";
+
+ public static final String JSTORM_CONF_DIR = "JSTORM_CONF_DIR";
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/GenericOptionsParser.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/GenericOptionsParser.java b/jstorm-core/src/main/java/backtype/storm/GenericOptionsParser.java
new file mode 100755
index 0000000..9319ce1
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/GenericOptionsParser.java
@@ -0,0 +1,283 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * <code>GenericOptionsParser</code> is a utility to parse command line arguments generic to Storm.
+ *
+ * <code>GenericOptionsParser</code> recognizes several standard command line arguments, enabling applications to easily specify additional jar files,
+ * configuration resources, data files etc.
+ *
+ * <h4 id="GenericOptions">Generic Options</h4>
+ *
+ * <p>
+ * The supported generic options are:
+ * </p>
+ * <p>
+ * <blockquote>
+ *
+ * <pre>
+ * -conf <conf.xml> load configurations from
+ * <conf.xml>
+ * -conf <conf.yaml> load configurations from
+ * <conf.yaml>
+ * -D <key=value> set <key> in configuration to
+ * <value> (preserve value's type)
+ * -libjars <comma separated list of jars> specify comma separated jars to be
+ * used by the submitted topology
+ * </pre>
+ *
+ * </blockquote>
+ * </p>
+ *
+ * <b>Note:</b> The XML configuration file specified by <code>-conf</code> shall be readable by Hadoop's <a href=
+ * "http://hadoop.apache.org/docs/current/api/org/apache/hadoop/conf/Configuration.html" ><code>Configuration</code></a> class. Also note that all configuration
+ * values of an XML file will be treated as strings, and <b>not as specific types</b>.
+ *
+ * <p>
+ * The general command line syntax is:
+ * </p>
+ * <p>
+ * <tt><pre>
+ * storm jar app.jar [genericOptions] [commandOptions]
+ * </pre></tt>
+ * </p>
+ *
+ * <p>
+ * Generic command line arguments <strong>might</strong> modify <code>Config</code> objects, given to constructors.
+ * </p>
+ *
+ * <h4>Configuration priority</h4>
+ *
+ * The following list defines the priorities of different configuration sources, in ascending order. Thus, if a configuration appears in more than one of them,
+ * only the last one will take effect.
+ *
+ * <ul>
+ * <li> <code>defaults.yaml</code> in classpath.
+ * <li> <code>storm.yaml</code> in classpath.
+ * <li>Configurations from files specified with the <code>-conf</code> option, in the order of appearance.
+ * <li>Configurations defined with the <code>-D</code> option, in order of appearance.
+ * </ul>
+ *
+ * <p>
+ * The functionality is implemented using Commons CLI.
+ * </p>
+ *
+ * @see Tool
+ * @see ToolRunner
+ */
+
+public class GenericOptionsParser {
+ static final Logger LOG = LoggerFactory.getLogger(GenericOptionsParser.class);
+
+ static final Charset UTF8 = Charset.forName("UTF-8");
+
+ public static final String TOPOLOGY_LIB_PATH = "topology.lib.path";
+
+ public static final String TOPOLOGY_LIB_NAME = "topology.lib.name";
+
+ Config conf;
+
+ CommandLine commandLine;
+
+ // Order in this map is important for these purposes:
+ // - configuration priority
+ static final LinkedHashMap<String, OptionProcessor> optionProcessors = new LinkedHashMap<String, OptionProcessor>();
+
+ public GenericOptionsParser(Config conf, String[] args) throws ParseException {
+ this(conf, new Options(), args);
+ }
+
+ public GenericOptionsParser(Config conf, Options options, String[] args) throws ParseException {
+ this.conf = conf;
+ parseGeneralOptions(options, conf, args);
+ }
+
+ public String[] getRemainingArgs() {
+ return commandLine.getArgs();
+ }
+
+ public Config getConfiguration() {
+ return conf;
+ }
+
+ static Options buildGeneralOptions(Options opts) {
+ Options r = new Options();
+
+ for (Object o : opts.getOptions())
+ r.addOption((Option) o);
+
+ Option libjars = OptionBuilder.withArgName("paths").hasArg().withDescription("comma separated jars to be used by the submitted topology").create("libjars");
+ r.addOption(libjars);
+ optionProcessors.put("libjars", new LibjarsProcessor());
+
+ Option conf = OptionBuilder.withArgName("configuration file").hasArg().withDescription("an application configuration file").create("conf");
+ r.addOption(conf);
+ optionProcessors.put("conf", new ConfFileProcessor());
+
+ // Must come after `conf': this option is of higher priority
+ Option extraConfig = OptionBuilder.withArgName("D").hasArg().withDescription("extra configurations (preserving types)").create("D");
+ r.addOption(extraConfig);
+ optionProcessors.put("D", new ExtraConfigProcessor());
+
+ return r;
+ }
+
+ void parseGeneralOptions(Options opts, Config conf, String[] args) throws ParseException {
+ opts = buildGeneralOptions(opts);
+ CommandLineParser parser = new GnuParser();
+ commandLine = parser.parse(opts, args, true);
+ processGeneralOptions(conf, commandLine);
+ }
+
+ void processGeneralOptions(Config conf, CommandLine commandLine) throws ParseException {
+ for (Map.Entry<String, OptionProcessor> e : optionProcessors.entrySet())
+ if (commandLine.hasOption(e.getKey()))
+ e.getValue().process(conf, commandLine);
+ }
+
+ static List<File> validateFiles(String pathList) throws IOException {
+ List<File> l = new ArrayList<File>();
+
+ for (String s : pathList.split(",")) {
+ File file = new File(s);
+ if (!file.exists())
+ throw new FileNotFoundException("File `" + file.getAbsolutePath() + "' does not exist");
+
+ l.add(file);
+ }
+
+ return l;
+ }
+
+ public static void printGenericCommandUsage(PrintStream out) {
+ String[] strs =
+ new String[] {
+ "Generic options supported are",
+ " -conf <conf.xml> load configurations from",
+ " <conf.xml>",
+ " -conf <conf.yaml> load configurations from",
+ " <conf.yaml>",
+ " -D <key>=<value> set <key> in configuration",
+ " to <value> (preserve value's type)",
+ " -libjars <comma separated list of jars> specify comma separated",
+ " jars to be used by",
+ " the submitted topology", };
+ for (String s : strs)
+ out.println(s);
+ }
+
+ static interface OptionProcessor {
+ public void process(Config conf, CommandLine commandLine) throws ParseException;
+ }
+
+ static class LibjarsProcessor implements OptionProcessor {
+ @Override
+ public void process(Config conf, CommandLine commandLine) throws ParseException {
+ try {
+ List<File> jarFiles = validateFiles(commandLine.getOptionValue("libjars"));
+ Map<String, String> jars = new HashMap<String, String>(jarFiles.size());
+ List<String> names = new ArrayList<String>(jarFiles.size());
+ for (File f : jarFiles) {
+ jars.put(f.getName(), f.getAbsolutePath());
+ names.add(f.getName());
+ }
+ conf.put(TOPOLOGY_LIB_PATH, jars);
+ conf.put(TOPOLOGY_LIB_NAME, names);
+
+ } catch (IOException e) {
+ throw new ParseException(e.getMessage());
+ }
+ }
+ }
+
+ static class ExtraConfigProcessor implements OptionProcessor {
+ static final Yaml yaml = new Yaml();
+
+ @Override
+ public void process(Config conf, CommandLine commandLine) throws ParseException {
+ for (String s : commandLine.getOptionValues("D")) {
+ String[] keyval = s.split("=", 2);
+ if (keyval.length != 2)
+ throw new ParseException("Invalid option value `" + s + "'");
+
+ conf.putAll((Map) yaml.load(keyval[0] + ": " + keyval[1]));
+ }
+ }
+ }
+
+ static class ConfFileProcessor implements OptionProcessor {
+ static final Yaml yaml = new Yaml();
+
+ static Map loadYamlConf(String f) throws IOException {
+ InputStreamReader reader = null;
+ try {
+ FileInputStream fis = new FileInputStream(f);
+ reader = new InputStreamReader(fis, UTF8);
+ return (Map) yaml.load(reader);
+ } finally {
+ if (reader != null)
+ reader.close();
+ }
+ }
+
+ static Map loadConf(String f) throws IOException {
+ if (f.endsWith(".yaml"))
+ return loadYamlConf(f);
+ throw new IOException("Unknown configuration file type: " + f + " does not end with either .yaml");
+ }
+
+ @Override
+ public void process(Config conf, CommandLine commandLine) throws ParseException {
+ try {
+ for (String f : commandLine.getOptionValues("conf")) {
+ Map m = loadConf(f);
+ if (m == null)
+ throw new ParseException("Empty configuration file " + f);
+ conf.putAll(m);
+ }
+ } catch (IOException e) {
+ throw new ParseException(e.getMessage());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/ICredentialsListener.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/ICredentialsListener.java b/jstorm-core/src/main/java/backtype/storm/ICredentialsListener.java
new file mode 100755
index 0000000..1a7bc1b
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/ICredentialsListener.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package backtype.storm;
+
+import java.util.Map;
+
+/**
+ * Allows a bolt or a spout to be informed when the credentials of the topology have changed.
+ */
+public interface ICredentialsListener {
+ /**
+ * Called when the credentials of a topology have changed.
+ * @param credentials the new credentials, could be null.
+ */
+ public void setCredentials(Map<String,String> credentials);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/ILocalCluster.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/ILocalCluster.java b/jstorm-core/src/main/java/backtype/storm/ILocalCluster.java
new file mode 100755
index 0000000..7d5aa35
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/ILocalCluster.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import backtype.storm.generated.AlreadyAliveException;
+import backtype.storm.generated.ClusterSummary;
+import backtype.storm.generated.InvalidTopologyException;
+import backtype.storm.generated.KillOptions;
+import backtype.storm.generated.SubmitOptions;
+import backtype.storm.generated.NotAliveException;
+import backtype.storm.generated.RebalanceOptions;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.generated.TopologyInfo;
+import backtype.storm.generated.Credentials;
+
+import java.util.Map;
+
+
+public interface ILocalCluster {
+ void submitTopology(String topologyName, Map conf, StormTopology topology) throws AlreadyAliveException, InvalidTopologyException;
+ void submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, SubmitOptions submitOpts) throws AlreadyAliveException, InvalidTopologyException;
+ void uploadNewCredentials(String topologyName, Credentials creds);
+ void killTopology(String topologyName) throws NotAliveException;
+ void killTopologyWithOpts(String name, KillOptions options) throws NotAliveException;
+ void activate(String topologyName) throws NotAliveException;
+ void deactivate(String topologyName) throws NotAliveException;
+ void rebalance(String name, RebalanceOptions options) throws NotAliveException;
+ void shutdown();
+ String getTopologyConf(String id);
+ StormTopology getTopology(String id);
+ ClusterSummary getClusterInfo();
+ TopologyInfo getTopologyInfo(String id);
+ Map getState();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/ILocalDRPC.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/ILocalDRPC.java b/jstorm-core/src/main/java/backtype/storm/ILocalDRPC.java
new file mode 100755
index 0000000..e478dca
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/ILocalDRPC.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import backtype.storm.daemon.Shutdownable;
+import backtype.storm.generated.DistributedRPC;
+import backtype.storm.generated.DistributedRPCInvocations;
+
+
+public interface ILocalDRPC extends DistributedRPC.Iface, DistributedRPCInvocations.Iface, Shutdownable {
+ public String getServiceId();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/LocalCluster.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/LocalCluster.java b/jstorm-core/src/main/java/backtype/storm/LocalCluster.java
new file mode 100755
index 0000000..b55bac4
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/LocalCluster.java
@@ -0,0 +1,275 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import java.util.Map;
+
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.generated.ClusterSummary;
+import backtype.storm.generated.Credentials;
+import backtype.storm.generated.KillOptions;
+import backtype.storm.generated.NotAliveException;
+import backtype.storm.generated.RebalanceOptions;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.generated.SubmitOptions;
+import backtype.storm.generated.TopologyInfo;
+import backtype.storm.utils.Utils;
+
+import com.alibaba.jstorm.utils.JStormUtils;
+
+public class LocalCluster implements ILocalCluster {
+
+ public static Logger LOG = LoggerFactory.getLogger(LocalCluster.class);
+
+ private LocalClusterMap state;
+
+ protected void setLogger() {
+ // the code is for log4j
+ // boolean needReset = true;
+ // Logger rootLogger = Logger.getRootLogger();
+ // if (rootLogger != null) {
+ // Enumeration appenders = rootLogger.getAllAppenders();
+ // if (appenders.hasMoreElements() == true) {
+ // needReset = false;
+ // }
+ // }
+ //
+ // if (needReset == true) {
+ // BasicConfigurator.configure();
+ // rootLogger.setLevel(Level.INFO);
+ // }
+
+ }
+
+ // this is easy to debug
+ protected static LocalCluster instance = null;
+
+ public static LocalCluster getInstance() {
+ return instance;
+ }
+
+ public LocalCluster() {
+ synchronized (LocalCluster.class) {
+ if (instance != null) {
+ throw new RuntimeException("LocalCluster should be single");
+ }
+ setLogger();
+
+ // fix in zk occur Address family not supported by protocol family:
+ // connect
+ System.setProperty("java.net.preferIPv4Stack", "true");
+
+ this.state = LocalUtils.prepareLocalCluster();
+ if (this.state == null)
+ throw new RuntimeException("prepareLocalCluster error");
+
+ instance = this;
+ }
+ }
+
+ @Override
+ public void submitTopology(String topologyName, Map conf, StormTopology topology) {
+ submitTopologyWithOpts(topologyName, conf, topology, null);
+ }
+
+ @Override
+ public void submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, SubmitOptions submitOpts) {
+ // TODO Auto-generated method stub
+ if (!Utils.isValidConf(conf))
+ throw new RuntimeException("Topology conf is not json-serializable");
+ JStormUtils.setLocalMode(true);
+
+ try {
+ if (submitOpts == null) {
+ state.getNimbus().submitTopology(topologyName, null, Utils.to_json(conf), topology);
+ } else {
+ state.getNimbus().submitTopologyWithOpts(topologyName, null, Utils.to_json(conf), topology, submitOpts);
+ }
+
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ LOG.error("Failed to submit topology " + topologyName, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void killTopology(String topologyName) {
+ // TODO Auto-generated method stub
+ try {
+ // kill topology quickly
+ KillOptions killOps = new KillOptions();
+ killOps.set_wait_secs(0);
+ state.getNimbus().killTopologyWithOpts(topologyName, killOps);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to kill Topology " + topologyName, e);
+ }
+ }
+
+ @Override
+ public void killTopologyWithOpts(String name, KillOptions options) throws NotAliveException {
+ // TODO Auto-generated method stub
+ try {
+ state.getNimbus().killTopologyWithOpts(name, options);
+ } catch (TException e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to kill Topology " + name, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void activate(String topologyName) {
+ // TODO Auto-generated method stub
+ try {
+ state.getNimbus().activate(topologyName);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to activate " + topologyName, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void deactivate(String topologyName) {
+ // TODO Auto-generated method stub
+ try {
+ state.getNimbus().deactivate(topologyName);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to deactivate " + topologyName, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void rebalance(String name, RebalanceOptions options) {
+ // TODO Auto-generated method stub
+ try {
+ state.getNimbus().rebalance(name, options);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to rebalance " + name, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ // TODO Auto-generated method stub
+ // in order to avoid kill topology's command competition
+ // it take 10 seconds to remove topology's node
+ JStormUtils.sleepMs(10 * 1000);
+ this.state.clean();
+ }
+
+ @Override
+ public String getTopologyConf(String id) {
+ // TODO Auto-generated method stub
+ try {
+ return state.getNimbus().getTopologyConf(id);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to get topology Conf of topologId: " + id, e);
+ }
+ return null;
+ }
+
+ @Override
+ public StormTopology getTopology(String id) {
+ // TODO Auto-generated method stub
+ try {
+ return state.getNimbus().getTopology(id);
+ } catch (NotAliveException e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to get topology of topologId: " + id, e);
+ } catch (TException e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to get topology of topologId: " + id, e);
+ }
+ return null;
+ }
+
+ @Override
+ public ClusterSummary getClusterInfo() {
+ // TODO Auto-generated method stub
+ try {
+ return state.getNimbus().getClusterInfo();
+ } catch (TException e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to get cluster info", e);
+ }
+ return null;
+ }
+
+ @Override
+ public TopologyInfo getTopologyInfo(String id) {
+ // TODO Auto-generated method stub
+ try {
+ return state.getNimbus().getTopologyInfo(id);
+ } catch (NotAliveException e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to get topology info of topologyId: " + id, e);
+ } catch (TException e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to get topology info of topologyId: " + id, e);
+ }
+ return null;
+ }
+
+ /***
+ * You should use getLocalClusterMap() to instead.This function will always return null
+ * */
+ @Deprecated
+ @Override
+ public Map getState() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ public LocalClusterMap getLocalClusterMap() {
+ return state;
+ }
+
+ public static void main(String[] args) throws Exception {
+ LocalCluster localCluster = null;
+ try {
+ localCluster = new LocalCluster();
+ } finally {
+ if (localCluster != null) {
+ localCluster.shutdown();
+ }
+ }
+ }
+
+ @Override
+ public void uploadNewCredentials(String topologyName, Credentials creds) {
+ // TODO Auto-generated method stub
+ try {
+ state.getNimbus().uploadNewCredentials(topologyName, creds);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ LOG.error("fail to uploadNewCredentials of topologyId: " + topologyName, e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/LocalClusterMap.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/LocalClusterMap.java b/jstorm-core/src/main/java/backtype/storm/LocalClusterMap.java
new file mode 100755
index 0000000..bd99c76
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/LocalClusterMap.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.jstorm.daemon.nimbus.NimbusServer;
+import com.alibaba.jstorm.daemon.nimbus.ServiceHandler;
+import com.alibaba.jstorm.daemon.supervisor.SupervisorManger;
+import com.alibaba.jstorm.utils.PathUtils;
+import com.alibaba.jstorm.zk.Factory;
+
+public class LocalClusterMap {
+
+ public static Logger LOG = LoggerFactory.getLogger(LocalClusterMap.class);
+
+ private NimbusServer nimbusServer;
+
+ private ServiceHandler nimbus;
+
+ private Factory zookeeper;
+
+ private Map conf;
+
+ private List<String> tmpDir;
+
+ private SupervisorManger supervisor;
+
+ public ServiceHandler getNimbus() {
+ return nimbus;
+ }
+
+ public void setNimbus(ServiceHandler nimbus) {
+ this.nimbus = nimbus;
+ }
+
+ public Factory getZookeeper() {
+ return zookeeper;
+ }
+
+ public void setZookeeper(Factory zookeeper) {
+ this.zookeeper = zookeeper;
+ }
+
+ public Map getConf() {
+ return conf;
+ }
+
+ public void setConf(Map conf) {
+ this.conf = conf;
+ }
+
+ public NimbusServer getNimbusServer() {
+ return nimbusServer;
+ }
+
+ public void setNimbusServer(NimbusServer nimbusServer) {
+ this.nimbusServer = nimbusServer;
+ }
+
+ public SupervisorManger getSupervisor() {
+ return supervisor;
+ }
+
+ public void setSupervisor(SupervisorManger supervisor) {
+ this.supervisor = supervisor;
+ }
+
+ public List<String> getTmpDir() {
+ return tmpDir;
+ }
+
+ public void setTmpDir(List<String> tmpDir) {
+ this.tmpDir = tmpDir;
+ }
+
+ public void clean() {
+
+ if (supervisor != null) {
+ supervisor.ShutdownAllWorkers();
+ supervisor.shutdown();
+ }
+
+ if (nimbusServer != null) {
+ nimbusServer.cleanup();
+ }
+
+ if (zookeeper != null)
+ zookeeper.shutdown();
+
+ // it will hava a problem:
+ // java.io.IOException: Unable to delete file:
+ // {TmpPath}\{UUID}\version-2\log.1
+ if (tmpDir != null) {
+ for (String dir : tmpDir) {
+ try {
+ PathUtils.rmr(dir);
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOG.error("Fail to delete " + dir);
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/LocalDRPC.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/LocalDRPC.java b/jstorm-core/src/main/java/backtype/storm/LocalDRPC.java
new file mode 100755
index 0000000..4113bf4
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/LocalDRPC.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.generated.DRPCRequest;
+import backtype.storm.utils.ServiceRegistry;
+
+import com.alibaba.jstorm.drpc.Drpc;
+
+public class LocalDRPC implements ILocalDRPC {
+ private static final Logger LOG = LoggerFactory.getLogger(LocalDRPC.class);
+
+ private Drpc handler = new Drpc();
+ private Thread thread;
+
+ private final String serviceId;
+
+ public LocalDRPC() {
+
+ thread = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ LOG.info("Begin to init local Drpc");
+ try {
+ handler.init();
+ } catch (Exception e) {
+ LOG.info("Failed to start local drpc");
+ System.exit(-1);
+ }
+ LOG.info("Successfully start local drpc");
+ }
+ });
+ thread.start();
+
+ serviceId = ServiceRegistry.registerService(handler);
+ }
+
+ @Override
+ public String execute(String functionName, String funcArgs) {
+ // TODO Auto-generated method stub
+ try {
+ return handler.execute(functionName, funcArgs);
+ } catch (Exception e) {
+ LOG.error("", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void result(String id, String result) throws TException {
+ // TODO Auto-generated method stub
+ handler.result(id, result);
+ }
+
+ @Override
+ public DRPCRequest fetchRequest(String functionName) throws TException {
+ // TODO Auto-generated method stub
+ return handler.fetchRequest(functionName);
+ }
+
+ @Override
+ public void failRequest(String id) throws TException {
+ // TODO Auto-generated method stub
+ handler.failRequest(id);
+ }
+
+ @Override
+ public void shutdown() {
+ // TODO Auto-generated method stub
+ ServiceRegistry.unregisterService(this.serviceId);
+ this.handler.shutdown();
+ }
+
+ @Override
+ public String getServiceId() {
+ // TODO Auto-generated method stub
+ return serviceId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/LocalUtils.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/LocalUtils.java b/jstorm-core/src/main/java/backtype/storm/LocalUtils.java
new file mode 100755
index 0000000..e32c07e
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/LocalUtils.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.messaging.IContext;
+import backtype.storm.utils.Utils;
+
+import com.alibaba.jstorm.client.ConfigExtension;
+import com.alibaba.jstorm.daemon.nimbus.DefaultInimbus;
+import com.alibaba.jstorm.daemon.nimbus.NimbusServer;
+import com.alibaba.jstorm.daemon.supervisor.Supervisor;
+import com.alibaba.jstorm.message.netty.NettyContext;
+import com.alibaba.jstorm.zk.Factory;
+import com.alibaba.jstorm.zk.Zookeeper;
+
+public class LocalUtils {
+
+ public static Logger LOG = LoggerFactory.getLogger(LocalUtils.class);
+
+ public static LocalClusterMap prepareLocalCluster() {
+ LocalClusterMap state = new LocalClusterMap();
+ try {
+ List<String> tmpDirs = new ArrayList();
+
+ String zkDir = getTmpDir();
+ tmpDirs.add(zkDir);
+ Factory zookeeper = startLocalZookeeper(zkDir);
+ Map conf = getLocalConf(zookeeper.getZooKeeperServer().getClientPort());
+
+ String nimbusDir = getTmpDir();
+ tmpDirs.add(nimbusDir);
+ Map nimbusConf = deepCopyMap(conf);
+ nimbusConf.put(Config.STORM_LOCAL_DIR, nimbusDir);
+ NimbusServer instance = new NimbusServer();
+
+ Map supervisorConf = deepCopyMap(conf);
+ String supervisorDir = getTmpDir();
+ tmpDirs.add(supervisorDir);
+ supervisorConf.put(Config.STORM_LOCAL_DIR, supervisorDir);
+ Supervisor supervisor = new Supervisor();
+ IContext context = getLocalContext(supervisorConf);
+
+ state.setNimbusServer(instance);
+ state.setNimbus(instance.launcherLocalServer(nimbusConf, new DefaultInimbus()));
+ state.setZookeeper(zookeeper);
+ state.setConf(conf);
+ state.setTmpDir(tmpDirs);
+ state.setSupervisor(supervisor.mkSupervisor(supervisorConf, context));
+ return state;
+ } catch (Exception e) {
+ LOG.error("prepare cluster error!", e);
+ state.clean();
+
+ }
+ return null;
+ }
+
+ private static Factory startLocalZookeeper(String tmpDir) {
+ for (int i = 2000; i < 65535; i++) {
+ try {
+ return Zookeeper.mkInprocessZookeeper(tmpDir, i);
+ } catch (Exception e) {
+ LOG.error("fail to launch zookeeper at port: " + i, e);
+ }
+ }
+ throw new RuntimeException("No port is available to launch an inprocess zookeeper.");
+ }
+
+ private static String getTmpDir() {
+ return System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID();
+ }
+
+ private static Map getLocalConf(int port) {
+ List<String> zkServers = new ArrayList<String>(1);
+ zkServers.add("localhost");
+ Map conf = Utils.readStormConfig();
+ conf.put(Config.STORM_CLUSTER_MODE, "local");
+ conf.put(Config.STORM_ZOOKEEPER_SERVERS, zkServers);
+ conf.put(Config.STORM_ZOOKEEPER_PORT, port);
+ conf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
+ conf.put(Config.ZMQ_LINGER_MILLIS, 0);
+ conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, false);
+ conf.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, 50);
+ ConfigExtension.setSpoutDelayRunSeconds(conf, 0);
+ ConfigExtension.setTaskCleanupTimeoutSec(conf, 0);
+ return conf;
+ }
+
+ private static IContext getLocalContext(Map conf) {
+ if (!(Boolean) conf.get(Config.STORM_LOCAL_MODE_ZMQ)) {
+ IContext result = new NettyContext();
+ ConfigExtension.setLocalWorkerPort(conf, 6800);
+ result.prepare(conf);
+ return result;
+ }
+ return null;
+ }
+
+ private static Map deepCopyMap(Map map) {
+ return new HashMap(map);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/StormSubmitter.java b/jstorm-core/src/main/java/backtype/storm/StormSubmitter.java
new file mode 100644
index 0000000..400875e
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/StormSubmitter.java
@@ -0,0 +1,366 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.generated.AlreadyAliveException;
+import backtype.storm.generated.InvalidTopologyException;
+import backtype.storm.generated.Nimbus;
+import backtype.storm.generated.NotAliveException;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.generated.SubmitOptions;
+import backtype.storm.generated.TopologyAssignException;
+import backtype.storm.utils.BufferFileInputStream;
+import backtype.storm.utils.NimbusClient;
+import backtype.storm.utils.Utils;
+
+/**
+ * Use this class to submit topologies to run on the Storm cluster. You should
+ * run your program with the "storm jar" command from the command-line, and then
+ * use this class to submit your topologies.
+ */
+public class StormSubmitter {
+ public static Logger LOG = LoggerFactory.getLogger(StormSubmitter.class);
+
+ private static Nimbus.Iface localNimbus = null;
+
+ public static void setLocalNimbus(Nimbus.Iface localNimbusHandler) {
+ StormSubmitter.localNimbus = localNimbusHandler;
+ }
+
+ /**
+ * Submits a topology to run on the cluster. A topology runs forever or
+ * until explicitly killed.
+ *
+ *
+ * @param name the name of the storm.
+ * @param stormConf the topology-specific configuration. See {@link Config}.
+ * @param topology the processing to execute.
+ * @throws AlreadyAliveException if a topology with this name is already
+ * running
+ * @throws InvalidTopologyException if an invalid topology was submitted
+ */
+ public static void submitTopology(String name, Map stormConf,
+ StormTopology topology) throws AlreadyAliveException,
+ InvalidTopologyException {
+ submitTopology(name, stormConf, topology, null);
+ }
+
+ public static void submitTopology(String name, Map stormConf,
+ StormTopology topology, SubmitOptions opts, List<File> jarFiles)
+ throws AlreadyAliveException, InvalidTopologyException {
+ if (jarFiles == null) {
+ jarFiles = new ArrayList<File>();
+ }
+ Map<String, String> jars = new HashMap<String, String>(jarFiles.size());
+ List<String> names = new ArrayList<String>(jarFiles.size());
+
+ for (File f : jarFiles) {
+ if (!f.exists()) {
+ LOG.info(f.getName() + " is not existed: "
+ + f.getAbsolutePath());
+ continue;
+ }
+ jars.put(f.getName(), f.getAbsolutePath());
+ names.add(f.getName());
+ }
+ LOG.info("Files: " + names + " will be loaded");
+ stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_PATH, jars);
+ stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_NAME, names);
+ submitTopology(name, stormConf, topology, opts);
+ }
+
+ public static void submitTopology(String name, Map stormConf,
+ StormTopology topology, SubmitOptions opts,
+ ProgressListener listener) throws AlreadyAliveException,
+ InvalidTopologyException {
+ submitTopology(name, stormConf, topology, opts);
+ }
+
+ /**
+ * Submits a topology to run on the cluster. A topology runs forever or
+ * until explicitly killed.
+ *
+ *
+ * @param name the name of the storm.
+ * @param stormConf the topology-specific configuration. See {@link Config}.
+ * @param topology the processing to execute.
+ * @param options to manipulate the starting of the topology
+ * @throws AlreadyAliveException if a topology with this name is already
+ * running
+ * @throws InvalidTopologyException if an invalid topology was submitted
+ */
+ public static void submitTopology(String name, Map stormConf,
+ StormTopology topology, SubmitOptions opts)
+ throws AlreadyAliveException, InvalidTopologyException {
+ if (!Utils.isValidConf(stormConf)) {
+ throw new IllegalArgumentException(
+ "Storm conf is not valid. Must be json-serializable");
+ }
+ stormConf = new HashMap(stormConf);
+ stormConf.putAll(Utils.readCommandLineOpts());
+ Map conf = Utils.readStormConfig();
+ conf.putAll(stormConf);
+ putUserInfo(conf, stormConf);
+ try {
+ String serConf = Utils.to_json(stormConf);
+ if (localNimbus != null) {
+ LOG.info("Submitting topology " + name + " in local mode");
+ localNimbus.submitTopology(name, null, serConf, topology);
+ } else {
+ NimbusClient client = NimbusClient.getConfiguredClient(conf);
+ try {
+ if (topologyNameExists(client, conf, name)) {
+ throw new RuntimeException("Topology with name `" + name
+ + "` already exists on cluster");
+ }
+
+ submitJar(client, conf);
+ LOG.info("Submitting topology " + name
+ + " in distributed mode with conf " + serConf);
+ if (opts != null) {
+ client.getClient().submitTopologyWithOpts(name, path,
+ serConf, topology, opts);
+ } else {
+ // this is for backwards compatibility
+ client.getClient().submitTopology(name, path, serConf,
+ topology);
+ }
+ } finally {
+ client.close();
+ }
+ }
+ LOG.info("Finished submitting topology: " + name);
+ } catch (InvalidTopologyException e) {
+ LOG.warn("Topology submission exception", e);
+ throw e;
+ } catch (AlreadyAliveException e) {
+ LOG.warn("Topology already alive exception", e);
+ throw e;
+ } catch (TopologyAssignException e) {
+ LOG.warn("Failed to assign " + e.get_msg(), e);
+ throw new RuntimeException(e);
+ } catch (TException e) {
+ LOG.warn("Failed to assign ", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Submits a topology to run on the cluster with a progress bar. A topology
+ * runs forever or until explicitly killed.
+ *
+ *
+ * @param name the name of the storm.
+ * @param stormConf the topology-specific configuration. See {@link Config}.
+ * @param topology the processing to execute.
+ * @throws AlreadyAliveException if a topology with this name is already
+ * running
+ * @throws InvalidTopologyException if an invalid topology was submitted
+ * @throws TopologyAssignException
+ */
+
+ public static void submitTopologyWithProgressBar(String name,
+ Map stormConf, StormTopology topology)
+ throws AlreadyAliveException, InvalidTopologyException {
+ submitTopologyWithProgressBar(name, stormConf, topology, null);
+ }
+
+ /**
+ * Submits a topology to run on the cluster with a progress bar. A topology
+ * runs forever or until explicitly killed.
+ *
+ *
+ * @param name the name of the storm.
+ * @param stormConf the topology-specific configuration. See {@link Config}.
+ * @param topology the processing to execute.
+ * @param opts to manipulate the starting of the topology
+ * @throws AlreadyAliveException if a topology with this name is already
+ * running
+ * @throws InvalidTopologyException if an invalid topology was submitted
+ * @throws TopologyAssignException
+ */
+
+ public static void submitTopologyWithProgressBar(String name,
+ Map stormConf, StormTopology topology, SubmitOptions opts)
+ throws AlreadyAliveException, InvalidTopologyException {
+
+ /**
+ * remove progress bar in jstorm
+ */
+ submitTopology(name, stormConf, topology, opts);
+ }
+
+ public static boolean topologyNameExists(NimbusClient client, Map conf, String name) {
+ if (StringUtils.isBlank(name)) {
+ throw new RuntimeException("TopologyName is empty");
+ }
+
+ try {
+ String topologyId = client.getClient().getTopologyId(name);
+ if (StringUtils.isBlank(topologyId) == false) {
+ return true;
+ }
+ return false;
+
+ } catch (NotAliveException e) {
+ return false;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static String submittedJar = null;
+ private static String path = null;
+
+ private static void submitJar(NimbusClient client, Map conf) {
+ if (submittedJar == null) {
+ try {
+ LOG.info("Jar not uploaded to master yet. Submitting jar...");
+ String localJar = System.getProperty("storm.jar");
+ path = client.getClient().beginFileUpload();
+ String[] pathCache = path.split("/");
+ String uploadLocation =
+ path + "/stormjar-" + pathCache[pathCache.length - 1]
+ + ".jar";
+ List<String> lib =
+ (List<String>) conf
+ .get(GenericOptionsParser.TOPOLOGY_LIB_NAME);
+ Map<String, String> libPath =
+ (Map<String, String>) conf
+ .get(GenericOptionsParser.TOPOLOGY_LIB_PATH);
+ if (lib != null && lib.size() != 0) {
+ for (String libName : lib) {
+ String jarPath = path + "/lib/" + libName;
+ client.getClient().beginLibUpload(jarPath);
+ submitJar(conf, libPath.get(libName), jarPath, client);
+ }
+
+ } else {
+ if (localJar == null) {
+ // no lib, no client jar
+ throw new RuntimeException(
+ "No client app jar, please upload it");
+ }
+ }
+
+ if (localJar != null) {
+ submittedJar =
+ submitJar(conf, localJar, uploadLocation, client);
+ } else {
+ // no client jar, but with lib jar
+ client.getClient().finishFileUpload(uploadLocation);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ LOG.info("Jar already uploaded to master. Not submitting jar.");
+ }
+ }
+
+ public static String submitJar(Map conf, String localJar,
+ String uploadLocation, NimbusClient client) {
+ if (localJar == null) {
+ throw new RuntimeException(
+ "Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
+ }
+
+ try {
+
+ LOG.info("Uploading topology jar " + localJar
+ + " to assigned location: " + uploadLocation);
+ int bufferSize = 512 * 1024;
+ Object maxBufSizeObject =
+ conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
+ if (maxBufSizeObject != null) {
+ bufferSize = Utils.getInt(maxBufSizeObject) / 2;
+ }
+
+ BufferFileInputStream is =
+ new BufferFileInputStream(localJar, bufferSize);
+ while (true) {
+ byte[] toSubmit = is.read();
+ if (toSubmit.length == 0)
+ break;
+ client.getClient().uploadChunk(uploadLocation,
+ ByteBuffer.wrap(toSubmit));
+ }
+ client.getClient().finishFileUpload(uploadLocation);
+ LOG.info("Successfully uploaded topology jar to assigned location: "
+ + uploadLocation);
+ return uploadLocation;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+
+ }
+ }
+
+ private static void putUserInfo(Map conf, Map stormConf) {
+ stormConf.put("user.group", conf.get("user.group"));
+ stormConf.put("user.name", conf.get("user.name"));
+ stormConf.put("user.password", conf.get("user.password"));
+ }
+
+ /**
+ * Interface use to track progress of file upload
+ */
+ public interface ProgressListener {
+ /**
+ * called before file is uploaded
+ *
+ * @param srcFile - jar file to be uploaded
+ * @param targetFile - destination file
+ * @param totalBytes - total number of bytes of the file
+ */
+ public void onStart(String srcFile, String targetFile, long totalBytes);
+
+ /**
+ * called whenever a chunk of bytes is uploaded
+ *
+ * @param srcFile - jar file to be uploaded
+ * @param targetFile - destination file
+ * @param bytesUploaded - number of bytes transferred so far
+ * @param totalBytes - total number of bytes of the file
+ */
+ public void onProgress(String srcFile, String targetFile,
+ long bytesUploaded, long totalBytes);
+
+ /**
+ * called when the file is uploaded
+ *
+ * @param srcFile - jar file to be uploaded
+ * @param targetFile - destination file
+ * @param totalBytes - total number of bytes of the file
+ */
+ public void onCompleted(String srcFile, String targetFile,
+ long totalBytes);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/Tool.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/Tool.java b/jstorm-core/src/main/java/backtype/storm/Tool.java
new file mode 100755
index 0000000..6722b24
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/Tool.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+/**
+ * A tool abstract class that supports handling of generic command-line options.
+ *
+ * <p>
+ * Here is how a typical <code>Tool</code> is implemented:
+ * </p>
+ * <p>
+ * <blockquote>
+ *
+ * <pre>
+ * public class TopologyApp extends Tool {
+ * {@literal @}Override
+ * public int run(String[] args) throws Exception {
+ * // Config processed by ToolRunner
+ * Config conf = getConf();
+ *
+ * // Other setups go here
+ * String name = "topology";
+ * StormTopology topology = buildTopology(args);
+ * StormSubmitter.submitTopology(name, conf, topology);
+ * return 0;
+ * }
+ *
+ * StormTopology buildTopology(String[] args) { ... }
+ *
+ * public static void main(String[] args) throws Exception {
+ * // Use ToolRunner to handle generic command-line options
+ * ToolRunner.run(new TopologyApp(), args);
+ * }
+ * }
+ * </pre>
+ *
+ * </blockquote>
+ * </p>
+ *
+ * @see GenericOptionsParser
+ * @see ToolRunner
+ */
+
+public abstract class Tool {
+ Config config;
+
+ public abstract int run(String[] args) throws Exception;
+
+ public Config getConf() {
+ return config;
+ }
+
+ public void setConf(Config config) {
+ this.config = config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/ToolRunner.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/ToolRunner.java b/jstorm-core/src/main/java/backtype/storm/ToolRunner.java
new file mode 100755
index 0000000..33f5034
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/ToolRunner.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.ParseException;
+
+import backtype.storm.utils.Utils;
+
+/**
+ * A utility to help run {@link Tool}s
+ *
+ * <p>
+ * <code>ToolRunner</code> can be used to run classes extending the <code>Tool</code> abstract class. It works in conjunction with {@link GenericOptionsParser} to parse the <a
+ * href="{@docRoot} to parse the <a href="{@docRoot} to parse the <a href="{@docRoot} to parse the <a
+ * href="{@docRoot}
+ * to parse the <a href="{@docRoot} to parse the <a href="{@docRoot} to parse the <a href="{@docRoot} to parse the <a href="{@docRoot}
+ * /backtype/storm/GenericOptionsParser.html#GenericOptions"> generic storm command line arguments</a> and modifies the <code>Config</code> of the
+ * <code>Tool</code>. The application-specific options are passed along without being modified.
+ *
+ * @see Tool
+ * @see GenericOptionsParser
+ */
+
+public class ToolRunner {
+ static final Logger LOG = LoggerFactory.getLogger(ToolRunner.class);
+
+ public static void run(Tool tool, String[] args) {
+ run(tool.getConf(), tool, args);
+ }
+
+ public static void run(Config conf, Tool tool, String[] args) {
+ try {
+ if (conf == null) {
+ conf = new Config();
+ conf.putAll(Utils.readStormConfig());
+ }
+
+ GenericOptionsParser parser = new GenericOptionsParser(conf, args);
+ tool.setConf(conf);
+
+ System.exit(tool.run(parser.getRemainingArgs()));
+ } catch (ParseException e) {
+ LOG.error("Error parsing generic options: {}", e.getMessage());
+ GenericOptionsParser.printGenericCommandUsage(System.err);
+ System.exit(2);
+ } catch (Exception e) {
+ LOG.error("Error running tool", e);
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/clojure/ClojureBolt.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/clojure/ClojureBolt.java b/jstorm-core/src/main/java/backtype/storm/clojure/ClojureBolt.java
new file mode 100755
index 0000000..5de9bde
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/clojure/ClojureBolt.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.clojure;
+
+import backtype.storm.coordination.CoordinatedBolt.FinishedCallback;
+import backtype.storm.generated.StreamInfo;
+import backtype.storm.task.IBolt;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.IRichBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.utils.Utils;
+import clojure.lang.IFn;
+import clojure.lang.PersistentArrayMap;
+import clojure.lang.Keyword;
+import clojure.lang.Symbol;
+import clojure.lang.RT;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+
+public class ClojureBolt implements IRichBolt, FinishedCallback {
+ Map<String, StreamInfo> _fields;
+ List<String> _fnSpec;
+ List<String> _confSpec;
+ List<Object> _params;
+
+ IBolt _bolt;
+
+ public ClojureBolt(List fnSpec, List confSpec, List<Object> params, Map<String, StreamInfo> fields) {
+ _fnSpec = fnSpec;
+ _confSpec = confSpec;
+ _params = params;
+ _fields = fields;
+ }
+
+ @Override
+ public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) {
+ IFn hof = Utils.loadClojureFn(_fnSpec.get(0), _fnSpec.get(1));
+ try {
+ IFn preparer = (IFn) hof.applyTo(RT.seq(_params));
+ final Map<Keyword,Object> collectorMap = new PersistentArrayMap( new Object[] {
+ Keyword.intern(Symbol.create("output-collector")), collector,
+ Keyword.intern(Symbol.create("context")), context});
+ List<Object> args = new ArrayList<Object>() {{
+ add(stormConf);
+ add(context);
+ add(collectorMap);
+ }};
+
+ _bolt = (IBolt) preparer.applyTo(RT.seq(args));
+ //this is kind of unnecessary for clojure
+ try {
+ _bolt.prepare(stormConf, context, collector);
+ } catch(AbstractMethodError ame) {
+
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ _bolt.execute(input);
+ }
+
+ @Override
+ public void cleanup() {
+ try {
+ _bolt.cleanup();
+ } catch(AbstractMethodError ame) {
+
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ for(String stream: _fields.keySet()) {
+ StreamInfo info = _fields.get(stream);
+ declarer.declareStream(stream, info.is_direct(), new Fields(info.get_output_fields()));
+ }
+ }
+
+ @Override
+ public void finishedId(Object id) {
+ if(_bolt instanceof FinishedCallback) {
+ ((FinishedCallback) _bolt).finishedId(id);
+ }
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ IFn hof = Utils.loadClojureFn(_confSpec.get(0), _confSpec.get(1));
+ try {
+ return (Map) hof.applyTo(RT.seq(_params));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/clojure/ClojureSpout.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/clojure/ClojureSpout.java b/jstorm-core/src/main/java/backtype/storm/clojure/ClojureSpout.java
new file mode 100755
index 0000000..f6422e3
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/clojure/ClojureSpout.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.clojure;
+
+import backtype.storm.generated.StreamInfo;
+import backtype.storm.spout.ISpout;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.utils.Utils;
+import clojure.lang.IFn;
+import clojure.lang.PersistentArrayMap;
+import clojure.lang.Keyword;
+import clojure.lang.Symbol;
+import clojure.lang.RT;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ClojureSpout implements IRichSpout {
+ Map<String, StreamInfo> _fields;
+ List<String> _fnSpec;
+ List<String> _confSpec;
+ List<Object> _params;
+
+ ISpout _spout;
+
+ public ClojureSpout(List fnSpec, List confSpec, List<Object> params, Map<String, StreamInfo> fields) {
+ _fnSpec = fnSpec;
+ _confSpec = confSpec;
+ _params = params;
+ _fields = fields;
+ }
+
+
+ @Override
+ public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
+ IFn hof = Utils.loadClojureFn(_fnSpec.get(0), _fnSpec.get(1));
+ try {
+ IFn preparer = (IFn) hof.applyTo(RT.seq(_params));
+ final Map<Keyword,Object> collectorMap = new PersistentArrayMap( new Object[] {
+ Keyword.intern(Symbol.create("output-collector")), collector,
+ Keyword.intern(Symbol.create("context")), context});
+ List<Object> args = new ArrayList<Object>() {{
+ add(conf);
+ add(context);
+ add(collectorMap);
+ }};
+
+ _spout = (ISpout) preparer.applyTo(RT.seq(args));
+ //this is kind of unnecessary for clojure
+ try {
+ _spout.open(conf, context, collector);
+ } catch(AbstractMethodError ame) {
+
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ _spout.close();
+ } catch(AbstractMethodError ame) {
+
+ }
+ }
+
+ @Override
+ public void nextTuple() {
+ try {
+ _spout.nextTuple();
+ } catch(AbstractMethodError ame) {
+
+ }
+
+ }
+
+ @Override
+ public void ack(Object msgId) {
+ try {
+ _spout.ack(msgId);
+ } catch(AbstractMethodError ame) {
+
+ }
+
+ }
+
+ @Override
+ public void fail(Object msgId) {
+ try {
+ _spout.fail(msgId);
+ } catch(AbstractMethodError ame) {
+
+ }
+
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ for(String stream: _fields.keySet()) {
+ StreamInfo info = _fields.get(stream);
+ declarer.declareStream(stream, info.is_direct(), new Fields(info.get_output_fields()));
+ }
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ IFn hof = Utils.loadClojureFn(_confSpec.get(0), _confSpec.get(1));
+ try {
+ return (Map) hof.applyTo(RT.seq(_params));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void activate() {
+ try {
+ _spout.activate();
+ } catch(AbstractMethodError ame) {
+
+ }
+ }
+
+ @Override
+ public void deactivate() {
+ try {
+ _spout.deactivate();
+ } catch(AbstractMethodError ame) {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/clojure/RichShellBolt.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/clojure/RichShellBolt.java b/jstorm-core/src/main/java/backtype/storm/clojure/RichShellBolt.java
new file mode 100755
index 0000000..a155008
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/clojure/RichShellBolt.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.clojure;
+
+import backtype.storm.generated.StreamInfo;
+import backtype.storm.task.ShellBolt;
+import backtype.storm.topology.IRichBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+public class RichShellBolt extends ShellBolt implements IRichBolt {
+ private Map<String, StreamInfo> _outputs;
+
+ public RichShellBolt(String[] command, Map<String, StreamInfo> outputs) {
+ super(command);
+ _outputs = outputs;
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ for(String stream: _outputs.keySet()) {
+ StreamInfo def = _outputs.get(stream);
+ if(def.is_direct()) {
+ declarer.declareStream(stream, true, new Fields(def.get_output_fields()));
+ } else {
+ declarer.declareStream(stream, new Fields(def.get_output_fields()));
+ }
+ }
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/clojure/RichShellSpout.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/clojure/RichShellSpout.java b/jstorm-core/src/main/java/backtype/storm/clojure/RichShellSpout.java
new file mode 100755
index 0000000..b49fbef
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/clojure/RichShellSpout.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.clojure;
+
+import backtype.storm.generated.StreamInfo;
+import backtype.storm.spout.ShellSpout;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+public class RichShellSpout extends ShellSpout implements IRichSpout {
+ private Map<String, StreamInfo> _outputs;
+
+ public RichShellSpout(String[] command, Map<String, StreamInfo> outputs) {
+ super(command);
+ _outputs = outputs;
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ for(String stream: _outputs.keySet()) {
+ StreamInfo def = _outputs.get(stream);
+ if(def.is_direct()) {
+ declarer.declareStream(stream, true, new Fields(def.get_output_fields()));
+ } else {
+ declarer.declareStream(stream, new Fields(def.get_output_fields()));
+ }
+ }
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/command/activate.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/command/activate.java b/jstorm-core/src/main/java/backtype/storm/command/activate.java
new file mode 100755
index 0000000..ed12e09
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/command/activate.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.command;
+
+import java.security.InvalidParameterException;
+import java.util.Map;
+
+import backtype.storm.utils.NimbusClient;
+import backtype.storm.utils.Utils;
+
+/**
+ * Activate topology
+ *
+ * @author longda
+ *
+ */
+public class activate {
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ // TODO Auto-generated method stub
+ if (args == null || args.length == 0) {
+ throw new InvalidParameterException("Should input topology name");
+ }
+
+ String topologyName = args[0];
+
+ NimbusClient client = null;
+ try {
+
+ Map conf = Utils.readStormConfig();
+ client = NimbusClient.getConfiguredClient(conf);
+
+ client.getClient().activate(topologyName);
+
+ System.out.println("Successfully submit command activate " + topologyName);
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ } finally {
+ if (client != null) {
+ client.close();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/command/config_value.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/command/config_value.java b/jstorm-core/src/main/java/backtype/storm/command/config_value.java
new file mode 100755
index 0000000..868ffdc
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/command/config_value.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.command;
+
+import java.security.InvalidParameterException;
+import java.util.Map;
+
+import backtype.storm.utils.NimbusClient;
+import backtype.storm.utils.Utils;
+
+/**
+ * Get configuration
+ *
+ * @author longda
+ *
+ */
+public class config_value {
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ // TODO Auto-generated method stub
+ if (args == null || args.length == 0) {
+ throw new InvalidParameterException("Should input key name");
+ }
+
+ String key = args[0];
+
+ Map conf = Utils.readStormConfig();
+
+ System.out.print("VALUE: " + String.valueOf(conf.get(key)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/command/deactivate.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/command/deactivate.java b/jstorm-core/src/main/java/backtype/storm/command/deactivate.java
new file mode 100755
index 0000000..22ac20d
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/command/deactivate.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.command;
+
+import java.security.InvalidParameterException;
+import java.util.Map;
+
+import backtype.storm.utils.NimbusClient;
+import backtype.storm.utils.Utils;
+
+/**
+ * Deactivate topology
+ *
+ * @author longda
+ *
+ */
+public class deactivate {
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ // TODO Auto-generated method stub
+ if (args == null || args.length == 0) {
+ throw new InvalidParameterException("Should input topology name");
+ }
+
+ String topologyName = args[0];
+
+ NimbusClient client = null;
+ try {
+
+ Map conf = Utils.readStormConfig();
+ client = NimbusClient.getConfiguredClient(conf);
+
+ client.getClient().deactivate(topologyName);
+
+ System.out.println("Successfully submit command deactivate " + topologyName);
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ } finally {
+ if (client != null) {
+ client.close();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e935da91/jstorm-core/src/main/java/backtype/storm/command/kill_topology.java
----------------------------------------------------------------------
diff --git a/jstorm-core/src/main/java/backtype/storm/command/kill_topology.java b/jstorm-core/src/main/java/backtype/storm/command/kill_topology.java
new file mode 100755
index 0000000..4ab3893
--- /dev/null
+++ b/jstorm-core/src/main/java/backtype/storm/command/kill_topology.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.command;
+
+import java.security.InvalidParameterException;
+import java.util.Map;
+
+import backtype.storm.generated.KillOptions;
+import backtype.storm.utils.NimbusClient;
+import backtype.storm.utils.Utils;
+
+/**
+ * Kill topology
+ *
+ * @author longda
+ *
+ */
+public class kill_topology {
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ // TODO Auto-generated method stub
+ if (args == null || args.length == 0) {
+ throw new InvalidParameterException("Should input topology name");
+ }
+
+ String topologyName = args[0];
+
+ NimbusClient client = null;
+ try {
+
+ Map conf = Utils.readStormConfig();
+ client = NimbusClient.getConfiguredClient(conf);
+
+ if (args.length == 1) {
+
+ client.getClient().killTopology(topologyName);
+ } else {
+ int delaySeconds = Integer.parseInt(args[1]);
+
+ KillOptions options = new KillOptions();
+ options.set_wait_secs(delaySeconds);
+
+ client.getClient().killTopologyWithOpts(topologyName, options);
+
+ }
+
+ System.out.println("Successfully submit command kill " + topologyName);
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ } finally {
+ if (client != null) {
+ client.close();
+ }
+ }
+ }
+
+}