You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2016/06/07 02:57:16 UTC

[1/4] storm git commit: STORM-1864 StormSubmitter should throw respective exceptions and log respective errors forregistered submitter hook invocation.

Repository: storm
Updated Branches:
  refs/heads/master a9ef86d00 -> 7108c7178


STORM-1864 StormSubmitter should throw respective exceptions and log respective errors forregistered submitter hook invocation.


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b5fe05a3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b5fe05a3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b5fe05a3

Branch: refs/heads/master
Commit: b5fe05a3115b8077e0105f70eee89edc3635eb2a
Parents: 3fe482e
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Wed May 25 11:51:12 2016 +0530
Committer: Satish Duggana <sd...@hortonworks.com>
Committed: Wed May 25 12:16:30 2016 +0530

----------------------------------------------------------------------
 .../storm/hive/trident/TridentHiveTopology.java | 29 ++++++++-----
 .../jvm/org/apache/storm/StormSubmitter.java    | 43 ++++++++++++++------
 .../storm/hooks/SubmitterHookException.java     | 41 +++++++++++++++++++
 3 files changed, 90 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/b5fe05a3/external/storm-hive/src/test/java/org/apache/storm/hive/trident/TridentHiveTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/test/java/org/apache/storm/hive/trident/TridentHiveTopology.java b/external/storm-hive/src/test/java/org/apache/storm/hive/trident/TridentHiveTopology.java
index d6c1d65..86a35e6 100644
--- a/external/storm-hive/src/test/java/org/apache/storm/hive/trident/TridentHiveTopology.java
+++ b/external/storm-hive/src/test/java/org/apache/storm/hive/trident/TridentHiveTopology.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -26,6 +26,7 @@ import org.apache.storm.Config;
 import org.apache.storm.LocalCluster;
 import org.apache.storm.StormSubmitter;
 import org.apache.storm.generated.StormTopology;
+import org.apache.storm.hooks.SubmitterHookException;
 import org.apache.storm.tuple.Fields;
 import org.apache.storm.tuple.Values;
 import org.apache.storm.task.TopologyContext;
@@ -41,9 +42,13 @@ import org.apache.storm.trident.Stream;
 import org.apache.storm.trident.TridentState;
 import org.apache.storm.trident.TridentTopology;
 import org.apache.storm.trident.state.StateFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 public class TridentHiveTopology {
+    private static final Logger LOG = LoggerFactory.getLogger(TridentHiveTopology.class);
+
     public static StormTopology buildTopology(String metaStoreURI, String dbName, String tblName, Object keytab, Object principal) {
         int batchSize = 100;
         FixedBatchSpout spout = new FixedBatchSpout(batchSize);
@@ -93,28 +98,32 @@ public class TridentHiveTopology {
         if(args.length == 3) {
             LocalCluster cluster = new LocalCluster();
             cluster.submitTopology("tridentHiveTopology", conf, buildTopology(metaStoreURI, dbName, tblName,null,null));
-            System.out.println("waiting for 60 seconds");
+            LOG.info("waiting for 60 seconds");
             waitForSeconds(60);
-            System.out.println("killing topology");
+            LOG.info("killing topology");
             cluster.killTopology("tridenHiveTopology");
-            System.out.println("cluster shutdown");
+            LOG.info("cluster shutdown");
             cluster.shutdown();
-            System.out.println("cluster shutdown");
+            LOG.info("cluster shutdown");
             System.exit(0);
         } else if(args.length == 4) {
             try {
                 StormSubmitter.submitTopology(args[3], conf, buildTopology(metaStoreURI, dbName, tblName,null,null));
-            } catch(Exception e) {
-                System.out.println("Failed to submit topology "+e);
+            } catch(SubmitterHookException e) {
+                LOG.warn("Topology is submitted but invoking ISubmitterHook failed", e);
+            } catch (Exception e) {
+                LOG.warn("Failed to submit topology ", e);
             }
         } else if (args.length == 6) {
             try {
                 StormSubmitter.submitTopology(args[3], conf, buildTopology(metaStoreURI, dbName, tblName,args[4],args[5]));
-            } catch(Exception e) {
-                System.out.println("Failed to submit topology "+e);
+            } catch(SubmitterHookException e) {
+                LOG.warn("Topology is submitted but invoking ISubmitterHook failed", e);
+            } catch (Exception e) {
+                LOG.warn("Failed to submit topology ", e);
             }
         } else {
-            System.out.println("Usage: TridentHiveTopology metastoreURI dbName tableName [topologyNamey]");
+            LOG.info("Usage: TridentHiveTopology metastoreURI dbName tableName [topologyNamey]");
         }
     }
 

http://git-wip-us.apache.org/repos/asf/storm/blob/b5fe05a3/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
index c199f97..077c577 100644
--- a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
@@ -24,6 +24,7 @@ import java.util.regex.Pattern;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.storm.hooks.SubmitterHookException;
 import org.apache.storm.scheduler.resource.ResourceUtils;
 import org.apache.storm.validation.ConfigValidation;
 import org.apache.commons.lang.StringUtils;
@@ -149,6 +150,7 @@ public class StormSubmitter {
      * @throws AlreadyAliveException if a topology with this name is already running
      * @throws InvalidTopologyException if an invalid topology was submitted
      * @throws AuthorizationException if authorization is failed
+     * @thorws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook}
      */
     public static void submitTopology(String name, Map stormConf, StormTopology topology)
             throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
@@ -166,6 +168,7 @@ public class StormSubmitter {
      * @throws AlreadyAliveException if a topology with this name is already running
      * @throws InvalidTopologyException if an invalid topology was submitted
      * @throws AuthorizationException if authorization is failed
+     * @thorws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook}
      */
     public static void submitTopology(String name, Map stormConf, StormTopology topology, SubmitOptions opts)
             throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
@@ -186,6 +189,7 @@ public class StormSubmitter {
      * @throws InvalidTopologyException
      * @throws AuthorizationException
      * @throws IllegalArgumentException thrown if configs will yield an unschedulable topology. validateConfs validates confs
+     * @thorws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook}
      */
     public static void submitTopologyAs(String name, Map stormConf, StormTopology topology, SubmitOptions opts, ProgressListener progressListener, String asUser)
             throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, IllegalArgumentException {
@@ -223,32 +227,30 @@ public class StormSubmitter {
                     // this is for backwards compatibility
                     localNimbus.submitTopology(name, stormConf, topology);
                 }
+                LOG.info("Finished submitting topology: " +  name);
             } else {
                 String serConf = JSONValue.toJSONString(stormConf);
-                NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser);
                 if(topologyNameExists(conf, name, asUser)) {
                     throw new RuntimeException("Topology with name `" + name + "` already exists on cluster");
                 }
                 String jar = submitJarAs(conf, System.getProperty("storm.jar"), progressListener, asUser);
-                try {
-                    LOG.info("Submitting topology " +  name + " in distributed mode with conf " + serConf);
-                    if(opts!=null) {
+                try (NimbusClient client = NimbusClient.getConfiguredClientAs(conf, asUser)) {
+                    LOG.info("Submitting topology " + name + " in distributed mode with conf " + serConf);
+                    if (opts != null) {
                         client.getClient().submitTopologyWithOpts(name, jar, serConf, topology, opts);
                     } else {
                         // this is for backwards compatibility
                         client.getClient().submitTopology(name, jar, serConf, topology);
                     }
-                } catch(InvalidTopologyException e) {
-                    LOG.warn("Topology submission exception: "+e.get_msg());
+                    LOG.info("Finished submitting topology: " + name);
+                } catch (InvalidTopologyException e) {
+                    LOG.warn("Topology submission exception: " + e.get_msg());
                     throw e;
-                } catch(AlreadyAliveException e) {
+                } catch (AlreadyAliveException e) {
                     LOG.warn("Topology already alive exception", e);
                     throw e;
-                } finally {
-                    client.close();
                 }
             }
-            LOG.info("Finished submitting topology: " +  name);
         } catch(TException e) {
             throw new RuntimeException(e);
         }
@@ -256,15 +258,29 @@ public class StormSubmitter {
 
     }
 
+    /**
+     *
+     * @param name
+     * @param asUser
+     * @param stormConf
+     * @param topology
+     *
+     * @thorws SubmitterHookException This is thrown when any Exception occurs during initialization or invocation of registered {@link ISubmitterHook}
+     */
     private static void invokeSubmitterHook(String name, String asUser, Map stormConf, StormTopology topology) {
+        String submissionNotifierClassName = null;
         try {
             if (stormConf.containsKey(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN)) {
-                ISubmitterHook submitterHook = (ISubmitterHook) Class.forName(stormConf.get(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN).toString()).newInstance();
+                submissionNotifierClassName = stormConf.get(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN).toString();
+                LOG.info("Initializing the registered ISubmitterHook [{}]", submissionNotifierClassName);
+                ISubmitterHook submitterHook = (ISubmitterHook) Class.forName(submissionNotifierClassName).newInstance();
                 TopologyInfo topologyInfo = Utils.getTopologyInfo(name, asUser, stormConf);
+                LOG.info("Invoking the registered ISubmitterHook [{}]", submissionNotifierClassName);
                 submitterHook.notify(topologyInfo, stormConf, topology);
             }
         } catch (Exception e) {
-            throw new RuntimeException(e);
+            LOG.warn("Error occurred in invoking submitter hook: "+submissionNotifierClassName, e);
+            throw new SubmitterHookException(e);
         }
     }
 
@@ -281,6 +297,7 @@ public class StormSubmitter {
      * @throws AlreadyAliveException if a topology with this name is already running
      * @throws InvalidTopologyException if an invalid topology was submitted
      * @throws AuthorizationException if authorization is failed
+     * @thorws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook}
      */
     @SuppressWarnings("unchecked")
     public static void submitTopology(String name, Map stormConf, StormTopology topology, SubmitOptions opts,
@@ -317,8 +334,8 @@ public class StormSubmitter {
      * @throws AlreadyAliveException if a topology with this name is already running
      * @throws InvalidTopologyException if an invalid topology was submitted
      * @throws AuthorizationException if authorization is failed
+     * @thorws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook}
      */
-
     public static void submitTopologyWithProgressBar(String name, Map stormConf, StormTopology topology, SubmitOptions opts) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
         // show a progress bar so we know we're not stuck (especially on slow connections)
         submitTopology(name, stormConf, topology, opts, new StormSubmitter.ProgressListener() {

http://git-wip-us.apache.org/repos/asf/storm/blob/b5fe05a3/storm-core/src/jvm/org/apache/storm/hooks/SubmitterHookException.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/hooks/SubmitterHookException.java b/storm-core/src/jvm/org/apache/storm/hooks/SubmitterHookException.java
new file mode 100644
index 0000000..b7679a7
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/hooks/SubmitterHookException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.storm.hooks;
+
+/**
+ * This Exception is thrown when registered {@link org.apache.storm.ISubmitterHook} could not be initialized or invoked.
+ */
+public class SubmitterHookException extends RuntimeException {
+
+    public SubmitterHookException() {
+    }
+
+    public SubmitterHookException(String message) {
+        super(message);
+    }
+
+    public SubmitterHookException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public SubmitterHookException(Throwable cause) {
+        super(cause);
+    }
+
+}


[2/4] storm git commit: STORM-1864 log messages with format

Posted by sr...@apache.org.
STORM-1864 log messages with format


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fbcd0ab7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fbcd0ab7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fbcd0ab7

Branch: refs/heads/master
Commit: fbcd0ab780b9c3047de2f522e4286ec5321cba09
Parents: b5fe05a
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Wed May 25 12:35:26 2016 +0530
Committer: Satish Duggana <sd...@hortonworks.com>
Committed: Wed May 25 15:01:25 2016 +0530

----------------------------------------------------------------------
 storm-core/src/jvm/org/apache/storm/StormSubmitter.java | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/fbcd0ab7/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
index 077c577..9766740 100644
--- a/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/org/apache/storm/StormSubmitter.java
@@ -273,13 +273,18 @@ public class StormSubmitter {
             if (stormConf.containsKey(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN)) {
                 submissionNotifierClassName = stormConf.get(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN).toString();
                 LOG.info("Initializing the registered ISubmitterHook [{}]", submissionNotifierClassName);
+
+                if(submissionNotifierClassName == null || submissionNotifierClassName.isEmpty()) {
+                    throw new IllegalArgumentException(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN + " property must be a non empty string.");
+                }
+
                 ISubmitterHook submitterHook = (ISubmitterHook) Class.forName(submissionNotifierClassName).newInstance();
                 TopologyInfo topologyInfo = Utils.getTopologyInfo(name, asUser, stormConf);
                 LOG.info("Invoking the registered ISubmitterHook [{}]", submissionNotifierClassName);
                 submitterHook.notify(topologyInfo, stormConf, topology);
             }
         } catch (Exception e) {
-            LOG.warn("Error occurred in invoking submitter hook: "+submissionNotifierClassName, e);
+            LOG.warn("Error occurred in invoking submitter hook:[{}] ",submissionNotifierClassName, e);
             throw new SubmitterHookException(e);
         }
     }


[3/4] storm git commit: Merge branch 'STORM-1864' of https://github.com/satishd/storm into STORM-1864

Posted by sr...@apache.org.
Merge branch 'STORM-1864' of https://github.com/satishd/storm into STORM-1864


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/87ab6b22
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/87ab6b22
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/87ab6b22

Branch: refs/heads/master
Commit: 87ab6b2287fddfcf3650daf5654e36cdae91ac4b
Parents: a9ef86d fbcd0ab
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Mon Jun 6 19:46:53 2016 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Mon Jun 6 19:46:53 2016 -0700

----------------------------------------------------------------------
 .../storm/hive/trident/TridentHiveTopology.java | 29 ++++++++----
 .../jvm/org/apache/storm/StormSubmitter.java    | 48 ++++++++++++++------
 .../storm/hooks/SubmitterHookException.java     | 41 +++++++++++++++++
 3 files changed, 95 insertions(+), 23 deletions(-)
----------------------------------------------------------------------



[4/4] storm git commit: Added STORM-1864 to CHANGELOG.

Posted by sr...@apache.org.
Added STORM-1864 to CHANGELOG.


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7108c717
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7108c717
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7108c717

Branch: refs/heads/master
Commit: 7108c71784f6b369a3bdc3bc5b7ffde336931f92
Parents: 87ab6b2
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Mon Jun 6 19:47:27 2016 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Mon Jun 6 19:47:27 2016 -0700

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/7108c717/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b58feab..0111752 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 2.0.0
+ * STORM-1864: StormSubmitter should throw respective exceptions and log respective errors forregistered submitter hook invocation
  * STORM-1766: A better algorithm server rack selection for RAS
  * STORM-1859: Ack late tuples in windowed mode
  * STORM-1851: Fix default nimbus impersonation authorizer config