You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2015/12/22 08:01:19 UTC

[1/5] incubator-apex-core git commit: APEX-103 - Add module and dag interface in API

Repository: incubator-apex-core
Updated Branches:
  refs/heads/devel-3 0d5bfa5be -> 47b3ce8e4


APEX-103 - Add module and dag interface in API


Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/2f1e1dfe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/2f1e1dfe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/2f1e1dfe

Branch: refs/heads/devel-3
Commit: 2f1e1dfeb400bd7f01275a95522766abe909d936
Parents: 0d5bfa5
Author: Vlad Rozov <v....@datatorrent.com>
Authored: Wed Sep 23 20:30:51 2015 -0700
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Mon Dec 21 23:41:04 2015 +0530

----------------------------------------------------------------------
 api/src/main/java/com/datatorrent/api/DAG.java  | 24 +++++++++++++++++
 .../main/java/com/datatorrent/api/Module.java   | 28 ++++++++++++++++++++
 .../stram/plan/logical/LogicalPlan.java         | 24 +++++++++++++++++
 3 files changed, 76 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/2f1e1dfe/api/src/main/java/com/datatorrent/api/DAG.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/DAG.java b/api/src/main/java/com/datatorrent/api/DAG.java
index 6b1d1b2..abe2954 100644
--- a/api/src/main/java/com/datatorrent/api/DAG.java
+++ b/api/src/main/java/com/datatorrent/api/DAG.java
@@ -20,6 +20,8 @@ package com.datatorrent.api;
 
 import java.io.Serializable;
 
+import org.apache.hadoop.classification.InterfaceStability;
+
 import com.datatorrent.api.Context.DAGContext;
 
 /**
@@ -157,6 +159,16 @@ public interface DAG extends DAGContext, Serializable
     public OutputPortMeta getMeta(Operator.OutputPort<?> port);
   }
 
+  @InterfaceStability.Evolving
+  interface ModuleMeta extends Serializable, Context
+  {
+    String getName();
+
+    InputPortMeta getMeta(Operator.InputPort<?> port);
+
+    OutputPortMeta getMeta(Operator.OutputPort<?> port);
+  }
+
   /**
    * Add new instance of operator under given name to the DAG.
    * The operator class must have a default constructor.
@@ -179,6 +191,12 @@ public interface DAG extends DAGContext, Serializable
    */
   public abstract <T extends Operator> T addOperator(String name, T operator);
 
+  @InterfaceStability.Evolving
+  <T extends Module> T addModule(String name, Class<T> moduleClass);
+
+  @InterfaceStability.Evolving
+  <T extends Module> T addModule(String name, T module);
+
   /**
    * <p>addStream.</p>
    * @param id Identifier of the stream that will be used to identify stream in DAG
@@ -256,9 +274,15 @@ public interface DAG extends DAGContext, Serializable
    */
   public abstract OperatorMeta getOperatorMeta(String operatorId);
 
+  @InterfaceStability.Evolving
+  ModuleMeta getModuleMeta(String moduleId);
+
   /**
    * <p>getMeta.</p>
    */
   public abstract OperatorMeta getMeta(Operator operator);
 
+  @InterfaceStability.Evolving
+  ModuleMeta getMeta(Module module);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/2f1e1dfe/api/src/main/java/com/datatorrent/api/Module.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Module.java b/api/src/main/java/com/datatorrent/api/Module.java
new file mode 100644
index 0000000..1220fc1
--- /dev/null
+++ b/api/src/main/java/com/datatorrent/api/Module.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.api;
+
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+@InterfaceStability.Evolving
+public interface Module
+{
+  void populateDAG(DAG dag, Configuration conf);
+}

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/2f1e1dfe/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 6405644..cca45d8 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -1079,6 +1079,18 @@ public class LogicalPlan implements Serializable, DAG
     return operator;
   }
 
+  @Override
+  public <T extends Module> T addModule(String name, Class<T> moduleClass)
+  {
+    throw new UnsupportedOperationException("Modules are not supported");
+  }
+
+  @Override
+  public <T extends Module> T addModule(String name, T module)
+  {
+    throw new UnsupportedOperationException("Modules are not supported");
+  }
+
   public void removeOperator(Operator operator)
   {
     OperatorMeta om = getMeta(operator);
@@ -1231,6 +1243,12 @@ public class LogicalPlan implements Serializable, DAG
   }
 
   @Override
+  public ModuleMeta getModuleMeta(String moduleName)
+  {
+    throw new UnsupportedOperationException("Modules are not supported");
+  }
+
+  @Override
   public OperatorMeta getMeta(Operator operator)
   {
     // TODO: cache mapping
@@ -1242,6 +1260,12 @@ public class LogicalPlan implements Serializable, DAG
     throw new IllegalArgumentException("Operator not associated with the DAG: " + operator);
   }
 
+  @Override
+  public ModuleMeta getMeta(Module module)
+  {
+    throw new UnsupportedOperationException("Modules are not supported");
+  }
+
   public int getMaxContainerCount()
   {
     return this.getValue(CONTAINERS_MAX_COUNT);


[2/5] incubator-apex-core git commit: APEXCORE-105 Introduce module meta Inject properties through xml file on modules.

Posted by th...@apache.org.
APEXCORE-105 Introduce module meta
Inject properties through xml file on modules.


Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/b0360d45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/b0360d45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/b0360d45

Branch: refs/heads/devel-3
Commit: b0360d45a361e462124db9c3000977987ca830e6
Parents: 2f1e1df
Author: Tushar R. Gosavi <tu...@apache.org>
Authored: Tue Oct 6 13:48:53 2015 +0530
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Tue Dec 22 01:16:51 2015 +0530

----------------------------------------------------------------------
 api/src/main/java/com/datatorrent/api/DAG.java  |   2 +
 .../stram/codec/LogicalPlanSerializer.java      |   2 +-
 .../stram/plan/logical/LogicalPlan.java         | 120 ++++++++++-
 .../plan/logical/LogicalPlanConfiguration.java  |  99 +++++++--
 .../stram/webapp/StramWebServices.java          |   2 +-
 .../logical/module/TestModuleProperties.java    |  58 +++++
 .../stram/plan/logical/module/TestModules.java  | 216 +++++++++++++++++++
 7 files changed, 472 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/api/src/main/java/com/datatorrent/api/DAG.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/DAG.java b/api/src/main/java/com/datatorrent/api/DAG.java
index abe2954..1dce402 100644
--- a/api/src/main/java/com/datatorrent/api/DAG.java
+++ b/api/src/main/java/com/datatorrent/api/DAG.java
@@ -164,6 +164,8 @@ public interface DAG extends DAGContext, Serializable
   {
     String getName();
 
+    Module getModule();
+
     InputPortMeta getMeta(Operator.InputPort<?> port);
 
     OutputPortMeta getMeta(Operator.OutputPort<?> port);

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
index 9e5ac04..90dd2b5 100644
--- a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
+++ b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
@@ -212,7 +212,7 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
       String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
       Operator operator = operatorMeta.getOperator();
       props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
-      BeanMap operatorProperties = LogicalPlanConfiguration.getOperatorProperties(operator);
+      BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
       @SuppressWarnings("rawtypes")
       Iterator entryIterator = operatorProperties.entryIterator();
       while (entryIterator.hasNext()) {

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index cca45d8..53e81bc 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
 import com.datatorrent.api.*;
@@ -48,7 +49,6 @@ import com.datatorrent.api.Operator.Unifier;
 import com.datatorrent.api.annotation.InputPortFieldAnnotation;
 import com.datatorrent.api.annotation.OperatorAnnotation;
 import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-
 import com.datatorrent.common.experimental.AppData;
 import com.datatorrent.common.metric.MetricsAggregator;
 import com.datatorrent.common.metric.SingleMetricAggregator;
@@ -1067,7 +1067,7 @@ public class LogicalPlan implements Serializable, DAG
   public <T extends Operator> T addOperator(String name, T operator)
   {
     if (operators.containsKey(name)) {
-      if (operators.get(name) == (Object)operator) {
+      if (operators.get(name).operator == operator) {
         return operator;
       }
       throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
@@ -1079,16 +1079,109 @@ public class LogicalPlan implements Serializable, DAG
     return operator;
   }
 
-  @Override
-  public <T extends Module> T addModule(String name, Class<T> moduleClass)
+  public final class ModuleMeta implements DAG.ModuleMeta, Serializable
   {
-    throw new UnsupportedOperationException("Modules are not supported");
+    private final LinkedHashMap<InputPortMeta, StreamMeta> inputStreams = new LinkedHashMap<>();
+    private final LinkedHashMap<OutputPortMeta, StreamMeta> outputStreams = new LinkedHashMap<>();
+    private final Attribute.AttributeMap attributes;
+    @SuppressWarnings("unused")
+    private final int id;
+    @NotNull
+    private final String name;
+    private transient Integer nindex; // for cycle detection
+    private transient Integer lowlink; // for cycle detection
+    private transient Module module;
+
+    public ModuleMeta(String name, Module module)
+    {
+      this(name, module, new DefaultAttributeMap());
+    }
+
+    public ModuleMeta(String name, Module module, DefaultAttributeMap attributeMap)
+    {
+      LOG.debug("Initializing {} as {}", name, module.getClass().getName());
+      this.name = name;
+      this.module = module;
+      this.id = logicalOperatorSequencer.decrementAndGet();
+      this.attributes = attributeMap;
+    }
+
+    @Override
+    public String getName()
+    {
+      return name;
+    }
+
+    @Override
+    public Module getModule()
+    {
+      return module;
+    }
+
+    @Override
+    public DAG.InputPortMeta getMeta(InputPort<?> port)
+    {
+      return null;
+    }
+
+    @Override
+    public DAG.OutputPortMeta getMeta(OutputPort<?> port)
+    {
+      return null;
+    }
+
+    @Override
+    public Attribute.AttributeMap getAttributes()
+    {
+      return null;
+    }
+
+    @Override
+    public <T> T getValue(Attribute<T> key)
+    {
+      return null;
+    }
+
+    @Override
+    public void setCounters(Object counters)
+    {
+
+    }
+
+    @Override
+    public void sendMetrics(Collection<String> metricNames)
+    {
+
+    }
   }
 
+  public transient Map<String, ModuleMeta> modules = Maps.newHashMap();
+
   @Override
   public <T extends Module> T addModule(String name, T module)
   {
-    throw new UnsupportedOperationException("Modules are not supported");
+    if (modules.containsKey(name)) {
+      if (modules.get(name).module == module) {
+        return module;
+      }
+      throw new IllegalArgumentException("duplicate module is: " + modules.get(name));
+    }
+    ModuleMeta meta = new ModuleMeta(name, module);
+    modules.put(name, meta);
+    return module;
+  }
+
+  @Override
+  public <T extends Module> T addModule(String name, Class<T> clazz)
+  {
+    T instance;
+    try {
+      instance = clazz.newInstance();
+    } catch (Exception ex) {
+      throw new IllegalArgumentException(ex);
+    }
+    addModule(name, instance);
+    return instance;
   }
 
   public void removeOperator(Operator operator)
@@ -1231,6 +1324,10 @@ public class LogicalPlan implements Serializable, DAG
     return Collections.unmodifiableCollection(this.operators.values());
   }
 
+  public Collection<ModuleMeta> getAllModules() {
+    return Collections.unmodifiableCollection(this.modules.values());
+  }
+
   public Collection<StreamMeta> getAllStreams()
   {
     return Collections.unmodifiableCollection(this.streams.values());
@@ -1242,10 +1339,9 @@ public class LogicalPlan implements Serializable, DAG
     return this.operators.get(operatorName);
   }
 
-  @Override
   public ModuleMeta getModuleMeta(String moduleName)
   {
-    throw new UnsupportedOperationException("Modules are not supported");
+    return null;
   }
 
   @Override
@@ -1260,10 +1356,14 @@ public class LogicalPlan implements Serializable, DAG
     throw new IllegalArgumentException("Operator not associated with the DAG: " + operator);
   }
 
-  @Override
   public ModuleMeta getMeta(Module module)
   {
-    throw new UnsupportedOperationException("Modules are not supported");
+    for (ModuleMeta m : getAllModules()) {
+      if (m.module == module) {
+        return m;
+      }
+    }
+    throw new IllegalArgumentException("Module not associated with the DAG: " + module);
   }
 
   public int getMaxContainerCount()

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 60bbdbe..9bbe85c 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -18,7 +18,6 @@
  */
 package com.datatorrent.stram.plan.logical;
 
-
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -32,7 +31,6 @@ import java.lang.reflect.Type;
 import java.util.*;
 import java.util.Map.Entry;
 
-
 import javax.validation.ValidationException;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -47,6 +45,7 @@ import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.commons.beanutils.BeanMap;
 import org.apache.commons.beanutils.BeanUtils;
 import org.apache.commons.collections.CollectionUtils;
@@ -61,10 +60,10 @@ import com.datatorrent.api.Context.DAGContext;
 import com.datatorrent.api.Context.OperatorContext;
 import com.datatorrent.api.Context.PortContext;
 import com.datatorrent.api.annotation.ApplicationAnnotation;
-
 import com.datatorrent.stram.StramUtils;
 import com.datatorrent.stram.client.StramClientUtils;
 import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.ModuleMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
@@ -2114,6 +2113,10 @@ public class LogicalPlanConfiguration {
     if (dag.getAttributes().get(Context.DAGContext.APPLICATION_NAME) == null) {
       dag.setAttribute(Context.DAGContext.APPLICATION_NAME, appName);
     }
+
+    // Expand the modules within the dag recursively
+    setModuleProperties(dag, appName);
+
     // inject external operator configuration
     setOperatorConfiguration(dag, appConfs, appName);
     setStreamConfiguration(dag, appConfs, appName);
@@ -2138,7 +2141,7 @@ public class LogicalPlanConfiguration {
   public Map<String, String> getProperties(OperatorMeta ow, String appName) {
     List<AppConf> appConfs = stramConf.getMatchingChildConf(appName, StramElement.APPLICATION);
     List<OperatorConf> opConfs = getMatchingChildConf(appConfs, ow.getName(), StramElement.OPERATOR);
-    return getProperties(ow, opConfs, appName);
+    return getProperties(getPropertyArgs(ow), opConfs, appName);
   }
 
   private Map<String,String> getApplicationProperties(List<AppConf> appConfs){
@@ -2153,17 +2156,17 @@ public class LogicalPlanConfiguration {
   /**
    * Get the configuration opProps for the given operator.
    * These can be operator specific settings or settings from matching templates.
-   * @param ow
+   * @param pa
    * @param opConfs
    * @param appName
    */
-  private Map<String, String> getProperties(OperatorMeta ow, List<OperatorConf> opConfs, String appName)
+  private Map<String, String> getProperties(PropertyArgs pa, List<OperatorConf> opConfs, String appName)
   {
     Map<String, String> opProps = Maps.newHashMap();
     Map<String, TemplateConf> templates = stramConf.getChildren(StramElement.TEMPLATE);
     // list of all templates that match operator, ordered by priority
     if (!templates.isEmpty()) {
-      TreeMap<Integer, TemplateConf> matchingTemplates = getMatchingTemplates(ow, appName, templates);
+      TreeMap<Integer, TemplateConf> matchingTemplates = getMatchingTemplates(pa, appName, templates);
       if (matchingTemplates != null && !matchingTemplates.isEmpty()) {
         // combined map of prioritized template settings
         for (TemplateConf t : matchingTemplates.descendingMap().values()) {
@@ -2197,23 +2200,46 @@ public class LogicalPlanConfiguration {
     return refTemplates;
   }
 
+  private static class PropertyArgs
+  {
+    String name;
+    String className;
+
+    public PropertyArgs(String name, String className)
+    {
+      this.name = name;
+      this.className = className;
+    }
+  }
+
+  private PropertyArgs getPropertyArgs(OperatorMeta om)
+  {
+    return new PropertyArgs(om.getName(), om.getOperator().getClass().getName());
+  }
+
+  private PropertyArgs getPropertyArgs(ModuleMeta mm)
+  {
+    return new PropertyArgs(mm.getName(), mm.getModule().getClass().getName());
+  }
+
   /**
    * Produce the collections of templates that apply for the given id.
-   * @param ow
+   * @param pa
    * @param appName
    * @param templates
    * @return TreeMap<Integer, TemplateConf>
    */
-  private TreeMap<Integer, TemplateConf> getMatchingTemplates(OperatorMeta ow, String appName, Map<String, TemplateConf> templates) {
+  private TreeMap<Integer, TemplateConf> getMatchingTemplates(PropertyArgs pa, String appName, Map<String, TemplateConf> templates)
+  {
     TreeMap<Integer, TemplateConf> tm = Maps.newTreeMap();
     for (TemplateConf t : templates.values()) {
-      if ((t.idRegExp != null && ow.getName().matches(t.idRegExp))) {
+      if ((t.idRegExp != null && pa.name.matches(t.idRegExp))) {
         tm.put(1, t);
       } else if (appName != null && t.appNameRegExp != null
           && appName.matches(t.appNameRegExp)) {
         tm.put(2, t);
       } else if (t.classNameRegExp != null
-          && ow.getOperator().getClass().getName().matches(t.classNameRegExp)) {
+          && pa.className.matches(t.classNameRegExp)) {
         tm.put(3, t);
       }
     }
@@ -2238,6 +2264,26 @@ public class LogicalPlanConfiguration {
     }
   }
 
+  /**
+   * Generic helper function to inject properties on the object.
+   *
+   * @param obj
+   * @param properties
+   * @param <T>
+   * @return
+   */
+  public static <T> T setObjectProperties(T obj, Map<String, String> properties)
+  {
+    try {
+      BeanUtils.populate(obj, properties);
+      return obj;
+    } catch (IllegalAccessException e) {
+      throw new IllegalArgumentException("Error setting operator properties", e);
+    } catch (InvocationTargetException e) {
+      throw new IllegalArgumentException("Error setting operator properties", e);
+    }
+  }
+
   public static StreamingApplication setApplicationProperties(StreamingApplication application, Map<String, String> properties)
   {
     try {
@@ -2249,9 +2295,9 @@ public class LogicalPlanConfiguration {
     }
   }
 
-  public static BeanMap getOperatorProperties(Operator operator)
+  public static BeanMap getObjectProperties(Object obj)
   {
-    return new BeanMap(operator);
+    return new BeanMap(obj);
   }
 
   /**
@@ -2266,12 +2312,26 @@ public class LogicalPlanConfiguration {
     List<AppConf> appConfs = stramConf.getMatchingChildConf(applicationName, StramElement.APPLICATION);
     for (OperatorMeta ow : dag.getAllOperators()) {
       List<OperatorConf> opConfs = getMatchingChildConf(appConfs, ow.getName(), StramElement.OPERATOR);
-      Map<String, String> opProps = getProperties(ow, opConfs, applicationName);
+      Map<String, String> opProps = getProperties(getPropertyArgs(ow), opConfs, applicationName);
       setOperatorProperties(ow.getOperator(), opProps);
     }
   }
 
   /**
+   * Set any properties from configuration on the modules in the DAG. This
+   * method may throw unchecked exception if the configuration contains
+   * properties that are invalid for a module.
+   *
+   * @param dag
+   * @param applicationName
+   */
+  public void setModuleProperties(LogicalPlan dag, String applicationName)
+  {
+    List<AppConf> appConfs = stramConf.getMatchingChildConf(applicationName, StramElement.APPLICATION);
+    setModuleConfiguration(dag, appConfs, applicationName);
+  }
+
+  /**
    * Set the application configuration.
    * @param dag
    * @param appName
@@ -2298,7 +2358,7 @@ public class LogicalPlanConfiguration {
       // Set the operator attributes
       setAttributes(opConfs, ow.getAttributes());
       // Set the operator opProps
-      Map<String, String> opProps = getProperties(ow, opConfs, appName);
+      Map<String, String> opProps = getProperties(getPropertyArgs(ow), opConfs, appName);
       setOperatorProperties(ow.getOperator(), opProps);
 
       // Set the port attributes
@@ -2327,6 +2387,15 @@ public class LogicalPlanConfiguration {
     }
   }
 
+  private void setModuleConfiguration(final LogicalPlan dag, List<AppConf> appConfs, String appName)
+  {
+    for (final ModuleMeta mw : dag.getAllModules()) {
+      List<OperatorConf> opConfs = getMatchingChildConf(appConfs, mw.getName(), StramElement.OPERATOR);
+      Map<String, String> opProps = getProperties(getPropertyArgs(mw), opConfs, appName);
+      setObjectProperties(mw.getModule(), opProps);
+    }
+  }
+
   private void setStreamConfiguration(LogicalPlan dag, List<AppConf> appConfs, String appAlias) {
     for (StreamMeta sm : dag.getAllStreams()) {
       List<StreamConf> smConfs = getMatchingChildConf(appConfs, sm.getName(), StramElement.STREAM);

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
index de085cd..6fdba00 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
@@ -715,7 +715,7 @@ public class StramWebServices
       throw new NotFoundException();
     }
 
-    BeanMap operatorProperties = LogicalPlanConfiguration.getOperatorProperties(logicalOperator.getOperator());
+    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalOperator.getOperator());
     Map<String, Object> m = new HashMap<String, Object>();
     @SuppressWarnings("rawtypes")
     Iterator entryIterator = operatorProperties.entryIterator();

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java
new file mode 100644
index 0000000..7951e26
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+
+public class TestModuleProperties
+{
+  @Test
+  public void testModuleProperties()
+  {
+    Configuration conf = new Configuration(false);
+    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.myStringProperty", "myStringPropertyValue");
+    conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.stringArrayField", "a,b,c");
+    conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty.key1", "key1Val");
+    conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key1.dot)", "key1dotVal");
+    conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key2.dot)", "key2dotVal");
+
+    LogicalPlan dag = new LogicalPlan();
+    TestModules.GenericModule o1 = dag.addModule("o1", new TestModules.GenericModule());
+    TestModules.ValidationTestModule o2 = dag.addModule("o2", new TestModules.ValidationTestModule());
+
+    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
+
+    pb.setModuleProperties(dag, "testSetOperatorProperties");
+    Assert.assertEquals("o1.myStringProperty", "myStringPropertyValue", o1.getMyStringProperty());
+    Assert.assertArrayEquals("o2.stringArrayField", new String[]{"a", "b", "c"}, o2.getStringArrayField());
+
+    Assert.assertEquals("o2.mapProperty.key1", "key1Val", o2.getMapProperty().get("key1"));
+    Assert.assertEquals("o2.mapProperty(key1.dot)", "key1dotVal", o2.getMapProperty().get("key1.dot"));
+    Assert.assertEquals("o2.mapProperty(key2.dot)", "key2dotVal", o2.getMapProperty().get("key2.dot"));
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java
new file mode 100644
index 0000000..8fad613
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import java.util.Map;
+
+import javax.validation.Valid;
+import javax.validation.constraints.AssertTrue;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Module;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.stram.engine.GenericOperatorProperty;
+
+public class TestModules
+{
+
+  public static class GenericModule implements Module
+  {
+    private static final Logger LOG = LoggerFactory.getLogger(TestModules.class);
+
+    public volatile Object inport1Tuple = null;
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient DefaultOutputPort<Object> outport2 = new DefaultOutputPort<Object>();
+
+    private String emitFormat;
+
+    public boolean booleanProperty;
+
+    private String myStringProperty;
+
+    private transient GenericOperatorProperty genericOperatorProperty = new GenericOperatorProperty("test");
+
+    public String getMyStringProperty()
+    {
+      return myStringProperty;
+    }
+
+    public void setMyStringProperty(String myStringProperty)
+    {
+      this.myStringProperty = myStringProperty;
+    }
+
+    public boolean isBooleanProperty()
+    {
+      return booleanProperty;
+    }
+
+    public void setBooleanProperty(boolean booleanProperty)
+    {
+      this.booleanProperty = booleanProperty;
+    }
+
+    public String propertySetterOnly;
+
+    /**
+     * setter w/o getter defined
+     *
+     * @param v
+     */
+    public void setStringPropertySetterOnly(String v)
+    {
+      this.propertySetterOnly = v;
+    }
+
+    public String getEmitFormat()
+    {
+      return emitFormat;
+    }
+
+    public void setEmitFormat(String emitFormat)
+    {
+      this.emitFormat = emitFormat;
+    }
+
+    public GenericOperatorProperty getGenericOperatorProperty()
+    {
+      return genericOperatorProperty;
+    }
+
+    public void setGenericOperatorProperty(GenericOperatorProperty genericOperatorProperty)
+    {
+      this.genericOperatorProperty = genericOperatorProperty;
+    }
+
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      LOG.debug("populateDAG of module called");
+    }
+  }
+
+  public static class ValidationTestModule implements Module
+  {
+    @NotNull
+    @Pattern(regexp = ".*malhar.*", message = "Value has to contain 'malhar'!")
+    private String stringField1;
+
+    @Min(2)
+    private int intField1;
+
+    @AssertTrue(message = "stringField1 should end with intField1")
+    private boolean isValidConfiguration()
+    {
+      return stringField1.endsWith(String.valueOf(intField1));
+    }
+
+    private String getterProperty2 = "";
+
+    @NotNull
+    public String getProperty2()
+    {
+      return getterProperty2;
+    }
+
+    public void setProperty2(String s)
+    {
+      // annotations need to be on the getter
+      getterProperty2 = s;
+    }
+
+    private String[] stringArrayField;
+
+    public String[] getStringArrayField()
+    {
+      return stringArrayField;
+    }
+
+    public void setStringArrayField(String[] stringArrayField)
+    {
+      this.stringArrayField = stringArrayField;
+    }
+
+    public class Nested
+    {
+      @NotNull
+      private String property = "";
+
+      public String getProperty()
+      {
+        return property;
+      }
+
+      public void setProperty(String property)
+      {
+        this.property = property;
+      }
+
+    }
+
+    @Valid
+    private final Nested nestedBean = new Nested();
+
+    private String stringProperty2;
+
+    public String getStringProperty2()
+    {
+      return stringProperty2;
+    }
+
+    public void setStringProperty2(String stringProperty2)
+    {
+      this.stringProperty2 = stringProperty2;
+    }
+
+    private Map<String, String> mapProperty = Maps.newHashMap();
+
+    public Map<String, String> getMapProperty()
+    {
+      return mapProperty;
+    }
+
+    public void setMapProperty(Map<String, String> mapProperty)
+    {
+      this.mapProperty = mapProperty;
+    }
+
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+
+    }
+  }
+
+}


[5/5] incubator-apex-core git commit: APEXCORE-144, APEXCORE-145 Rest api changes to view module information

Posted by th...@apache.org.
APEXCORE-144, APEXCORE-145 Rest api changes to view module information


Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/47b3ce8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/47b3ce8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/47b3ce8e

Branch: refs/heads/devel-3
Commit: 47b3ce8e4019f325c2e91968cb120763f18bb174
Parents: c1314ea
Author: shubham <sh...@github.com>
Authored: Tue Nov 17 12:10:08 2015 +0530
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Tue Dec 22 12:03:55 2015 +0530

----------------------------------------------------------------------
 .../stram/StreamingContainerManager.java        |  20 +++
 .../java/com/datatorrent/stram/cli/DTCli.java   |   8 +-
 .../stram/codec/LogicalPlanSerializer.java      |  47 ++++++-
 .../stram/plan/logical/LogicalPlan.java         |   7 +-
 .../plan/logical/LogicalPlanConfiguration.java  |   4 +-
 .../stram/webapp/StramWebServices.java          | 123 +++++++++++++------
 6 files changed, 158 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
index 29c6a2c..162245b 100644
--- a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
+++ b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
@@ -95,6 +95,7 @@ import com.datatorrent.stram.engine.WindowGenerator;
 import com.datatorrent.stram.plan.logical.LogicalOperatorStatus;
 import com.datatorrent.stram.plan.logical.LogicalPlan;
 import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.ModuleMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
@@ -2246,6 +2247,25 @@ public class StreamingContainerManager implements PlanContext
     return fillLogicalOperatorInfo(operatorMeta);
   }
 
+  public ModuleMeta getModuleMeta(String moduleName)
+  {
+    return getModuleMeta(moduleName, getLogicalPlan());
+  }
+
+  private ModuleMeta getModuleMeta(String moduleName, LogicalPlan dag)
+  {
+    for (ModuleMeta m : dag.getAllModules()) {
+      if (m.getFullName().equals(moduleName)) {
+        return m;
+      }
+      ModuleMeta res = getModuleMeta(moduleName, m.getDag());
+      if (res != null) {
+        return res;
+      }
+    }
+    return null;
+  }
+
   public List<LogicalOperatorInfo> getLogicalOperatorInfoList()
   {
     List<LogicalOperatorInfo> infoList = new ArrayList<LogicalOperatorInfo>();

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java b/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
index deb0967..696f497 100644
--- a/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
+++ b/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
@@ -2824,7 +2824,7 @@ public class DTCli
               }
               LogicalPlan logicalPlan = appFactory.createApp(submitApp.getLogicalPlanConfiguration());
               map.put("applicationName", appFactory.getName());
-              map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan));
+              map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan, false));
             } finally {
               if (raw) {
                 System.setOut(originalStream);
@@ -2840,7 +2840,7 @@ public class DTCli
             LogicalPlan logicalPlan = appFactory.createApp(submitApp.getLogicalPlanConfiguration());
             Map<String, Object> map = new HashMap<String, Object>();
             map.put("applicationName", appFactory.getName());
-            map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan));
+            map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan, false));
             printJson(map);
           } else if (filename.endsWith(".properties")) {
             File file = new File(filename);
@@ -2849,7 +2849,7 @@ public class DTCli
             LogicalPlan logicalPlan = appFactory.createApp(submitApp.getLogicalPlanConfiguration());
             Map<String, Object> map = new HashMap<String, Object>();
             map.put("applicationName", appFactory.getName());
-            map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan));
+            map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan, false));
             printJson(map);
           } else {
             StramAppLauncher submitApp = getStramAppLauncher(filename, config, commandLineInfo.ignorePom);
@@ -2893,7 +2893,7 @@ public class DTCli
               Map<String, Object> map = new HashMap<String, Object>();
               map.put("applicationName", appInfo.name);
               if (appInfo.dag != null) {
-                map.put("logicalPlan", LogicalPlanSerializer.convertToMap(appInfo.dag));
+                map.put("logicalPlan", LogicalPlanSerializer.convertToMap(appInfo.dag, false));
               }
               if (appInfo.error != null) {
                 map.put("error", appInfo.error);

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
index 90dd2b5..7b61d5b 100644
--- a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
+++ b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
@@ -88,7 +88,7 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
    * @param dag
    * @return
    */
-  public static Map<String, Object> convertToMap(LogicalPlan dag)
+  public static Map<String, Object> convertToMap(LogicalPlan dag, boolean includeModules)
   {
     HashMap<String, Object> result = new HashMap<String, Object>();
     ArrayList<Object> operatorArray = new ArrayList< Object>();
@@ -200,6 +200,15 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
         streamDetailMap.put("locality", streamMeta.getLocality().name());
       }
     }
+
+    if (includeModules) {
+      ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
+      result.put("modules", modulesArray);
+      for(LogicalPlan.ModuleMeta meta : dag.getAllModules()) {
+        modulesArray.add(getLogicalModuleDetails(dag, meta));
+      }
+    }
+
     return result;
   }
 
@@ -323,13 +332,43 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
 
   public static JSONObject convertToJsonObject(LogicalPlan dag)
   {
-    return new JSONObject(convertToMap(dag));
+    return new JSONObject(convertToMap(dag, false));
   }
 
   @Override
-  public void serialize(LogicalPlan dag, JsonGenerator jg, SerializerProvider sp) throws IOException, JsonProcessingException
+  public void serialize(LogicalPlan dag, JsonGenerator jg, SerializerProvider sp) throws IOException,
+      JsonProcessingException
   {
-    jg.writeObject(convertToMap(dag));
+    jg.writeObject(convertToMap(dag, false));
+  }
+
+  /**
+   * Return information about operators and inner modules of a module.
+   *
+   * @param dag        top level DAG
+   * @param moduleMeta module information. DAG within module is used for constructing response.
+   * @return
+   */
+  private static Map<String, Object> getLogicalModuleDetails(LogicalPlan dag, LogicalPlan.ModuleMeta moduleMeta)
+  {
+    Map<String, Object> moduleDetailMap = new HashMap<String, Object>();
+    ArrayList<String> operatorArray = new ArrayList<>();
+    moduleDetailMap.put("name", moduleMeta.getName());
+    moduleDetailMap.put("className", moduleMeta.getModule().getClass().getName());
+
+    moduleDetailMap.put("operators", operatorArray);
+    for (OperatorMeta operatorMeta : moduleMeta.getDag().getAllOperators()) {
+      if (operatorMeta.getModuleName() == null) {
+        String fullName = moduleMeta.getFullName() + LogicalPlan.MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName();
+        operatorArray.add(fullName);
+      }
+    }
+    ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
+    moduleDetailMap.put("modules", modulesArray);
+    for (LogicalPlan.ModuleMeta meta : moduleMeta.getDag().getAllModules()) {
+      modulesArray.add(getLogicalModuleDetails(dag, meta));
+    }
+    return moduleDetailMap;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 21039cc..377fa6d 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -32,6 +32,7 @@ import javax.validation.constraints.NotNull;
 
 import org.apache.commons.lang.builder.ToStringBuilder;
 import org.apache.commons.lang.builder.ToStringStyle;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.slf4j.Logger;
@@ -1090,9 +1091,9 @@ public class LogicalPlan implements Serializable, DAG
     }
 
     // Avoid name conflict with module.
-    if (modules.containsKey(name))
+    if (modules.containsKey(name)) {
       throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
-
+    }
     OperatorMeta decl = new OperatorMeta(name, operator);
     rootOperators.add(decl); // will be removed when a sink is added to an input port for this operator
     operators.put(name, decl);
@@ -1193,7 +1194,7 @@ public class LogicalPlan implements Serializable, DAG
      * @param parentDAG parent dag to populate with operators from this and inner modules.
      * @param conf      configuration object.
      */
-    public void flattenModule(LogicalPlan parentDAG, org.apache.hadoop.conf.Configuration conf)
+    public void flattenModule(LogicalPlan parentDAG, Configuration conf)
     {
       module.populateDAG(dag, conf);
       for (ModuleMeta subModuleMeta : dag.getAllModules()) {

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 483576a..dbd3bc3 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -18,6 +18,7 @@
  */
 package com.datatorrent.stram.plan.logical;
 
+
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -31,6 +32,7 @@ import java.lang.reflect.Type;
 import java.util.*;
 import java.util.Map.Entry;
 
+
 import javax.validation.ValidationException;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -45,7 +47,6 @@ import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
 import org.apache.commons.beanutils.BeanMap;
 import org.apache.commons.beanutils.BeanUtils;
 import org.apache.commons.collections.CollectionUtils;
@@ -60,6 +61,7 @@ import com.datatorrent.api.Context.DAGContext;
 import com.datatorrent.api.Context.OperatorContext;
 import com.datatorrent.api.Context.PortContext;
 import com.datatorrent.api.annotation.ApplicationAnnotation;
+
 import com.datatorrent.stram.StramUtils;
 import com.datatorrent.stram.client.StramClientUtils;
 import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
index 6fdba00..fd47d35 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
@@ -68,6 +68,7 @@ import com.datatorrent.stram.StreamingContainerManager;
 import com.datatorrent.stram.StringCodecs;
 import com.datatorrent.stram.codec.LogicalPlanSerializer;
 import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlan.ModuleMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
 import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
 import com.datatorrent.stram.plan.logical.requests.LogicalPlanRequest;
@@ -561,17 +562,14 @@ public class StramWebServices
         LOG.debug("Setting property for {}: {}={}", operatorName, key, val);
         dagManager.setOperatorProperty(operatorName, key, val);
       }
-    }
-    catch (JSONException ex) {
+    } catch (JSONException ex) {
       LOG.warn("Got JSON Exception: ", ex);
-    }
-    catch (Exception ex) {
+    } catch (Exception ex) {
       LOG.error("Caught exception: ", ex);
       throw new RuntimeException(ex);
     }
     return response;
   }
-
   @POST // not supported by WebAppProxyServlet, can only be called directly
   @Path(PATH_PHYSICAL_PLAN_OPERATORS + "/{operatorId:\\d+}/properties")
   @Consumes(MediaType.APPLICATION_JSON)
@@ -633,60 +631,97 @@ public class StramWebServices
   public JSONObject getPorts(@PathParam("operatorName") String operatorName)
   {
     OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
+    Set<LogicalPlan.InputPortMeta> inputPorts;
+    Set<LogicalPlan.OutputPortMeta> outputPorts;
     if (logicalOperator == null) {
-      throw new NotFoundException();
+      ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
+      if (logicalModule == null) {
+        throw new NotFoundException();
+      }
+      inputPorts = logicalModule.getInputStreams().keySet();
+      outputPorts = logicalModule.getOutputStreams().keySet();
+    } else {
+      inputPorts = logicalOperator.getInputStreams().keySet();
+      outputPorts = logicalOperator.getOutputStreams().keySet();
     }
+
+    JSONObject result = getPortsObjects(inputPorts, outputPorts);
+    return result;
+  }
+
+  private JSONObject getPortsObjects(Collection<LogicalPlan.InputPortMeta> inputs, Collection<LogicalPlan.OutputPortMeta> outputs)
+  {
     JSONObject result = new JSONObject();
     JSONArray ports = new JSONArray();
     try {
-      for (LogicalPlan.InputPortMeta inputPort : logicalOperator.getInputStreams().keySet()) {
+      for (LogicalPlan.InputPortMeta inputPort : inputs) {
         JSONObject port = new JSONObject();
         port.put("name", inputPort.getPortName());
         port.put("type", "input");
         ports.put(port);
       }
-      for (LogicalPlan.OutputPortMeta outputPort : logicalOperator.getOutputStreams().keySet()) {
+      for (LogicalPlan.OutputPortMeta outputPort : outputs) {
         JSONObject port = new JSONObject();
         port.put("name", outputPort.getPortName());
         port.put("type", "output");
         ports.put(port);
       }
       result.put("ports", ports);
-    }
-    catch (JSONException ex) {
+    } catch (JSONException ex) {
       throw new RuntimeException(ex);
     }
     return result;
   }
 
+  private JSONObject getPortObject(Collection<LogicalPlan.InputPortMeta> inputs, Collection<LogicalPlan.OutputPortMeta> outputs,
+                                   String portName) throws JSONException
+  {
+    for (LogicalPlan.InputPortMeta inputPort : inputs) {
+      if (inputPort.getPortName().equals(portName)) {
+        JSONObject port = new JSONObject();
+        port.put("name", inputPort.getPortName());
+        port.put("type", "input");
+        return port;
+      }
+    }
+    for (LogicalPlan.OutputPortMeta outputPort : outputs) {
+      if (outputPort.getPortName().equals(portName)) {
+        JSONObject port = new JSONObject();
+        port.put("name", outputPort.getPortName());
+        port.put("type", "output");
+        return port;
+      }
+    }
+    return null;
+  }
+
+
   @GET
   @Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports/{portName}")
   @Produces(MediaType.APPLICATION_JSON)
   public JSONObject getPort(@PathParam("operatorName") String operatorName, @PathParam("portName") String portName)
   {
     OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
+    Set<LogicalPlan.InputPortMeta> inputPorts;
+    Set<LogicalPlan.OutputPortMeta> outputPorts;
     if (logicalOperator == null) {
-      throw new NotFoundException();
+      ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
+      if (logicalModule == null) {
+        throw new NotFoundException();
+      }
+      inputPorts = logicalModule.getInputStreams().keySet();
+      outputPorts = logicalModule.getOutputStreams().keySet();
+    } else {
+      inputPorts = logicalOperator.getInputStreams().keySet();
+      outputPorts = logicalOperator.getOutputStreams().keySet();
     }
+
     try {
-      for (LogicalPlan.InputPortMeta inputPort : logicalOperator.getInputStreams().keySet()) {
-        if (portName.equals(portName)) {
-          JSONObject port = new JSONObject();
-          port.put("name", inputPort.getPortName());
-          port.put("type", "input");
-          return port;
-        }
-      }
-      for (LogicalPlan.OutputPortMeta outputPort : logicalOperator.getOutputStreams().keySet()) {
-        if (portName.equals(portName)) {
-          JSONObject port = new JSONObject();
-          port.put("name", outputPort.getPortName());
-          port.put("type", "output");
-          return port;
-        }
+      JSONObject resp = getPortObject(inputPorts, outputPorts, portName);
+      if (resp != null) {
+        return resp;
       }
-    }
-    catch (JSONException ex) {
+    } catch (JSONException ex) {
       throw new RuntimeException(ex);
     }
     throw new NotFoundException();
@@ -711,31 +746,41 @@ public class StramWebServices
   {
     init();
     OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
+    BeanMap operatorProperties = null;
     if (logicalOperator == null) {
-      throw new NotFoundException();
+      ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
+      if (logicalModule == null) {
+        throw new NotFoundException();
+      }
+      operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalModule.getModule());
+    } else {
+      operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalOperator.getOperator());
     }
 
-    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalOperator.getOperator());
+    Map<String, Object> m = getPropertiesAsMap(propertyName, operatorProperties);
+    return new JSONObject(objectMapper.writeValueAsString(m));
+  }
+
+  private Map<String, Object> getPropertiesAsMap(@QueryParam("propertyName") String propertyName, BeanMap operatorProperties)
+  {
     Map<String, Object> m = new HashMap<String, Object>();
     @SuppressWarnings("rawtypes")
     Iterator entryIterator = operatorProperties.entryIterator();
     while (entryIterator.hasNext()) {
       try {
         @SuppressWarnings("unchecked")
-        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
+        Entry<String, Object> entry = (Entry<String, Object>)entryIterator.next();
         if (propertyName == null) {
           m.put(entry.getKey(), entry.getValue());
-        }
-        else if (propertyName.equals(entry.getKey())) {
+        } else if (propertyName.equals(entry.getKey())) {
           m.put(entry.getKey(), entry.getValue());
           break;
         }
-      }
-      catch (Exception ex) {
+      } catch (Exception ex) {
         LOG.warn("Caught exception", ex);
       }
     }
-    return new JSONObject(objectMapper.writeValueAsString(m));
+    return m;
   }
 
   @GET
@@ -765,10 +810,10 @@ public class StramWebServices
   @GET
   @Path(PATH_LOGICAL_PLAN)
   @Produces(MediaType.APPLICATION_JSON)
-  public JSONObject getLogicalPlan() throws JSONException, IOException
+  public JSONObject getLogicalPlan(@QueryParam("includeModules") String includeModules) throws JSONException, IOException
   {
-    LogicalPlan lp = dagManager.getLogicalPlan();
-    return new JSONObject(objectMapper.writeValueAsString(LogicalPlanSerializer.convertToMap(lp)));
+    return new JSONObject(objectMapper.writeValueAsString(LogicalPlanSerializer.convertToMap(
+        dagManager.getLogicalPlan(), includeModules != null)));
   }
 
   @POST // not supported by WebAppProxyServlet, can only be called directly


[3/5] incubator-apex-core git commit: APEXCORE-104 Added flattening of module into parent DAG

Posted by th...@apache.org.
APEXCORE-104 Added flattening of module into parent DAG


Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/14a09bb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/14a09bb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/14a09bb5

Branch: refs/heads/devel-3
Commit: 14a09bb51cb30585e7979ec022ec762ac2ba91e5
Parents: b0360d4
Author: chinmaykolhatkar <ch...@datatorrent.com>
Authored: Wed Oct 7 15:06:36 2015 +0530
Committer: chinmaykolhatkar <ch...@datatorrent.com>
Committed: Tue Dec 22 01:42:43 2015 +0530

----------------------------------------------------------------------
 api/src/main/java/com/datatorrent/api/DAG.java  |   4 -
 .../stram/plan/logical/LogicalPlan.java         | 169 +++++++++++++++----
 .../plan/logical/LogicalPlanConfiguration.java  |   8 +
 3 files changed, 146 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/14a09bb5/api/src/main/java/com/datatorrent/api/DAG.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/DAG.java b/api/src/main/java/com/datatorrent/api/DAG.java
index 1dce402..7c793f9 100644
--- a/api/src/main/java/com/datatorrent/api/DAG.java
+++ b/api/src/main/java/com/datatorrent/api/DAG.java
@@ -165,10 +165,6 @@ public interface DAG extends DAGContext, Serializable
     String getName();
 
     Module getModule();
-
-    InputPortMeta getMeta(Operator.InputPort<?> port);
-
-    OutputPortMeta getMeta(Operator.OutputPort<?> port);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/14a09bb5/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 53e81bc..5a3e167 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -80,6 +80,7 @@ public class LogicalPlan implements Serializable, DAG
   public static final String SER_FILE_NAME = "dt-conf.ser";
   public static final String LAUNCH_CONFIG_FILE_NAME = "dt-launch-config.xml";
   private static final transient AtomicInteger logicalOperatorSequencer = new AtomicInteger();
+  public static final String MODULE_NAMESPACE_SEPARATOR = "$";
 
   /**
    * Constant
@@ -146,6 +147,7 @@ public class LogicalPlan implements Serializable, DAG
 
   private final Map<String, StreamMeta> streams = new HashMap<String, StreamMeta>();
   private final Map<String, OperatorMeta> operators = new HashMap<String, OperatorMeta>();
+  public final Map<String, ModuleMeta> modules = new LinkedHashMap<>();
   private final List<OperatorMeta> rootOperators = new ArrayList<OperatorMeta>();
   private final Attribute.AttributeMap attributes = new DefaultAttributeMap();
   private transient int nodeIndex = 0; // used for cycle validation
@@ -733,6 +735,7 @@ public class LogicalPlan implements Serializable, DAG
     private transient Integer lowlink; // for cycle detection
     private transient Operator operator;
     private MetricAggregatorMeta metricAggregatorMeta;
+    private String moduleName;  // Name of the module which has this operator. null if this is a top level operator.
 
     /*
      * Used for  OIO validation,
@@ -819,6 +822,16 @@ public class LogicalPlan implements Serializable, DAG
       return metricAggregatorMeta;
     }
 
+    public String getModuleName()
+    {
+      return moduleName;
+    }
+
+    public void setModuleName(String moduleName)
+    {
+      this.moduleName = moduleName;
+    }
+
     protected void populateAggregatorMeta()
     {
       AutoMetric.Aggregator aggregator = getValue(OperatorContext.METRICS_AGGREGATOR);
@@ -1073,37 +1086,38 @@ public class LogicalPlan implements Serializable, DAG
       throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
     }
 
+    // Avoid name conflict with module.
+    if (modules.containsKey(name))
+      throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
+
     OperatorMeta decl = new OperatorMeta(name, operator);
     rootOperators.add(decl); // will be removed when a sink is added to an input port for this operator
     operators.put(name, decl);
     return operator;
   }
 
+  /**
+   * Module meta object.
+   */
   public final class ModuleMeta implements DAG.ModuleMeta, Serializable
   {
     private final LinkedHashMap<InputPortMeta, StreamMeta> inputStreams = new LinkedHashMap<>();
     private final LinkedHashMap<OutputPortMeta, StreamMeta> outputStreams = new LinkedHashMap<>();
     private final Attribute.AttributeMap attributes;
-    @SuppressWarnings("unused")
-    private final int id;
     @NotNull
-    private final String name;
-    private transient Integer nindex; // for cycle detection
-    private transient Integer lowlink; // for cycle detection
+    private String name;
     private transient Module module;
+    private ModuleMeta parent;
+    private LogicalPlan dag = null;
+    private transient String fullName;
 
-    public ModuleMeta(String name, Module module)
-    {
-      this(name, module, new DefaultAttributeMap());
-    }
-
-    public ModuleMeta(String name, Module module, DefaultAttributeMap attributeMap)
+    private ModuleMeta(String name, Module module)
     {
       LOG.debug("Initializing {} as {}", name, module.getClass().getName());
       this.name = name;
       this.module = module;
-      this.id = logicalOperatorSequencer.decrementAndGet();
-      this.attributes = attributeMap;
+      this.attributes = new DefaultAttributeMap();
+      this.dag = new LogicalPlan();
     }
 
     @Override
@@ -1119,43 +1133,104 @@ public class LogicalPlan implements Serializable, DAG
     }
 
     @Override
-    public DAG.InputPortMeta getMeta(InputPort<?> port)
+    public Attribute.AttributeMap getAttributes()
     {
-      return null;
+      return attributes;
     }
 
     @Override
-    public DAG.OutputPortMeta getMeta(OutputPort<?> port)
+    public <T> T getValue(Attribute<T> key)
     {
-      return null;
+      return attributes.get(key);
     }
 
     @Override
-    public Attribute.AttributeMap getAttributes()
+    public void setCounters(Object counters)
     {
-      return null;
+
     }
 
     @Override
-    public <T> T getValue(Attribute<T> key)
+    public void sendMetrics(Collection<String> metricNames)
     {
-      return null;
+
     }
 
-    @Override
-    public void setCounters(Object counters)
+    public LinkedHashMap<InputPortMeta, StreamMeta> getInputStreams()
     {
+      return inputStreams;
+    }
 
+    public LinkedHashMap<OutputPortMeta, StreamMeta> getOutputStreams()
+    {
+      return outputStreams;
     }
 
-    @Override
-    public void sendMetrics(Collection<String> metricNames)
+    public LogicalPlan getDag()
     {
+      return dag;
+    }
 
+    private void writeObject(ObjectOutputStream out) throws IOException
+    {
+      out.defaultWriteObject();
+      FSStorageAgent.store(out, module);
     }
-  }
 
-  public transient Map<String, ModuleMeta> modules = Maps.newHashMap();
+    private void readObject(ObjectInputStream input) throws IOException, ClassNotFoundException
+    {
+      input.defaultReadObject();
+      module = (Module)FSStorageAgent.retrieve(input);
+    }
+
+    /**
+     * Expand the module and add its operator to the parentDAG. After this method finishes the module is expanded fully
+     * with all its submodules also expanded. The parentDAG contains the operator added by all the modules.
+     *
+     * @param parentDAG parent dag to populate with operators from this and inner modules.
+     * @param conf      configuration object.
+     */
+    public void flattenModule(LogicalPlan parentDAG, org.apache.hadoop.conf.Configuration conf)
+    {
+      module.populateDAG(dag, conf);
+      for (ModuleMeta subModuleMeta : dag.getAllModules()) {
+        subModuleMeta.setParent(this);
+        subModuleMeta.flattenModule(dag, conf);
+      }
+      parentDAG.addDAGToCurrentDAG(this);
+    }
+
+    /**
+     * Return full name of the module. If this is a inner module, i.e module inside of module this method will traverse
+     * till the top level module, and construct the name by concatenating name of modules in the chain in reverse order
+     * separated by MODULE_NAMESPACE_SEPARATO.
+     *
+     * For example If there is module M1, which adds another module M2 in the DAG. Then the full name of the module M2
+     * is ("M1" ++ MODULE_NAMESPACE_SEPARATO + "M2")
+     *
+     * @return full name of the module.
+     */
+    public String getFullName()
+    {
+      if (fullName != null) {
+        return fullName;
+      }
+
+      if (parent == null) {
+        fullName = name;
+      } else {
+        fullName = parent.getFullName() + MODULE_NAMESPACE_SEPARATOR + name;
+      }
+      return fullName;
+    }
+
+    private void setParent(ModuleMeta meta)
+    {
+      this.parent = meta;
+    }
+
+    private static final long serialVersionUID = 7562277769188329223L;
+  }
 
   @Override
   public <T extends Module> T addModule(String name, T module)
@@ -1166,6 +1241,10 @@ public class LogicalPlan implements Serializable, DAG
       }
       throw new IllegalArgumentException("duplicate module is: " + modules.get(name));
     }
+    if (operators.containsKey(name)) {
+      throw new IllegalArgumentException("duplicate module is: " + modules.get(name));
+    }
+
     ModuleMeta meta = new ModuleMeta(name, module);
     modules.put(name, meta);
     return module;
@@ -1228,6 +1307,33 @@ public class LogicalPlan implements Serializable, DAG
     return s;
   }
 
+  @SuppressWarnings({ "unchecked", "rawtypes" })
+  private void addDAGToCurrentDAG(ModuleMeta moduleMeta)
+  {
+    LogicalPlan subDag = moduleMeta.getDag();
+    String subDAGName = moduleMeta.getName();
+    String name;
+    for (OperatorMeta operatorMeta : subDag.getAllOperators()) {
+      name = subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName();
+      this.addOperator(name, operatorMeta.getOperator());
+      OperatorMeta operatorMetaNew = this.getOperatorMeta(name);
+      operatorMetaNew.setModuleName(operatorMeta.getModuleName() == null ? subDAGName :
+          subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getModuleName());
+    }
+
+    for (StreamMeta streamMeta : subDag.getAllStreams()) {
+      OutputPortMeta sourceMeta = streamMeta.getSource();
+      List<InputPort<?>> ports = new LinkedList<>();
+      for (InputPortMeta inputPortMeta : streamMeta.getSinks()) {
+        ports.add(inputPortMeta.getPortObject());
+      }
+      InputPort[] inputPorts = ports.toArray(new InputPort[]{});
+
+      name = subDAGName + MODULE_NAMESPACE_SEPARATOR + streamMeta.getName();
+      this.addStream(name, sourceMeta.getPortObject(), inputPorts);
+    }
+  }
+
   @Override
   @SuppressWarnings("unchecked")
   public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1)
@@ -1276,7 +1382,7 @@ public class LogicalPlan implements Serializable, DAG
 
   private OutputPortMeta assertGetPortMeta(Operator.OutputPort<?> port)
   {
-    for (OperatorMeta o: getAllOperators()) {
+    for (OperatorMeta o : getAllOperators()) {
       OutputPortMeta opm = o.getPortMapping().outPortMap.get(port);
       if (opm != null) {
         return opm;
@@ -1287,7 +1393,7 @@ public class LogicalPlan implements Serializable, DAG
 
   private InputPortMeta assertGetPortMeta(Operator.InputPort<?> port)
   {
-    for (OperatorMeta o: getAllOperators()) {
+    for (OperatorMeta o : getAllOperators()) {
       InputPortMeta opm = o.getPortMapping().inPortMap.get(port);
       if (opm != null) {
         return opm;
@@ -1324,7 +1430,8 @@ public class LogicalPlan implements Serializable, DAG
     return Collections.unmodifiableCollection(this.operators.values());
   }
 
-  public Collection<ModuleMeta> getAllModules() {
+  public Collection<ModuleMeta> getAllModules()
+  {
     return Collections.unmodifiableCollection(this.modules.values());
   }
 
@@ -1341,7 +1448,7 @@ public class LogicalPlan implements Serializable, DAG
 
   public ModuleMeta getModuleMeta(String moduleName)
   {
-    return null;
+    return this.modules.get(moduleName);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/14a09bb5/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 9bbe85c..6dc4c0c 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -2116,12 +2116,20 @@ public class LogicalPlanConfiguration {
 
     // Expand the modules within the dag recursively
     setModuleProperties(dag, appName);
+    flattenDAG(dag, conf);
 
     // inject external operator configuration
     setOperatorConfiguration(dag, appConfs, appName);
     setStreamConfiguration(dag, appConfs, appName);
   }
 
+  private void flattenDAG(LogicalPlan dag, Configuration conf)
+  {
+    for (ModuleMeta moduleMeta : dag.getAllModules()) {
+      moduleMeta.flattenModule(dag, conf);
+    }
+  }
+
   public static Properties readProperties(String filePath) throws IOException
   {
     InputStream is = new FileInputStream(filePath);


[4/5] incubator-apex-core git commit: APEXCORE-194 Added support for proxy ports Added test cases.

Posted by th...@apache.org.
APEXCORE-194 Added support for proxy ports
Added test cases.


Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/c1314eaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/c1314eaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/c1314eaf

Branch: refs/heads/devel-3
Commit: c1314eafaac239b420d085a4584d5c5acaf3e69b
Parents: 14a09bb
Author: bhupeshchawda <bh...@gmail.com>
Authored: Tue Oct 6 12:34:24 2015 +0530
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Tue Dec 22 02:04:18 2015 +0530

----------------------------------------------------------------------
 .../main/java/com/datatorrent/api/Module.java   | 120 ++++
 .../stram/plan/logical/LogicalPlan.java         |  51 +-
 .../plan/logical/LogicalPlanConfiguration.java  |   1 +
 .../plan/logical/module/ModuleAppTest.java      | 168 ++++++
 .../logical/module/TestModuleExpansion.java     | 552 +++++++++++++++++++
 5 files changed, 888 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/api/src/main/java/com/datatorrent/api/Module.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Module.java b/api/src/main/java/com/datatorrent/api/Module.java
index 1220fc1..67682e7 100644
--- a/api/src/main/java/com/datatorrent/api/Module.java
+++ b/api/src/main/java/com/datatorrent/api/Module.java
@@ -21,8 +21,128 @@ package com.datatorrent.api;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.Operator.InputPort;
+import com.datatorrent.api.Operator.OutputPort;
+import com.datatorrent.api.Operator.Unifier;
+
+/**
+ * A Module is a component which can be added to the DAG similar to the operator,
+ * using addModule API. The module should implement populateDAG method, which
+ * will be called by the platform, and DAG populated by the module will be
+ * replace in place of the module.
+ *
+ */
 @InterfaceStability.Evolving
 public interface Module
 {
   void populateDAG(DAG dag, Configuration conf);
+
+  /**
+   * These ports allow platform to short circuit module port to the operator port. i.e When a module is expanded, it can
+   * specify  which operator's port is used to replaced the module port in the final DAG.
+   *
+   * @param <T> data type accepted at the input port.
+   */
+  final class ProxyInputPort<T> implements InputPort<T>
+  {
+    InputPort<T> inputPort;
+
+    public void set(InputPort<T> port)
+    {
+      inputPort = port;
+    }
+
+    public InputPort<T> get()
+    {
+      return inputPort;
+    }
+
+    @Override
+    public void setup(PortContext context)
+    {
+      if (inputPort != null) {
+        inputPort.setup(context);
+      }
+    }
+
+    @Override
+    public void teardown()
+    {
+      if (inputPort != null) {
+        inputPort.teardown();
+      }
+    }
+
+    @Override
+    public Sink<T> getSink()
+    {
+      return inputPort == null ? null : inputPort.getSink();
+    }
+
+    @Override
+    public void setConnected(boolean connected)
+    {
+      if (inputPort != null) {
+        inputPort.setConnected(connected);
+      }
+    }
+
+    @Override
+    public StreamCodec<T> getStreamCodec()
+    {
+      return inputPort == null ? null : inputPort.getStreamCodec();
+    }
+  }
+
+  /**
+   * Similar to ProxyInputPort, but on output side.
+   *
+   * @param <T> datatype emitted on the port.
+   */
+  final class ProxyOutputPort<T> implements OutputPort<T>
+  {
+    OutputPort<T> outputPort;
+
+    public void set(OutputPort<T> port)
+    {
+      outputPort = port;
+    }
+
+    public OutputPort<T> get()
+    {
+      return outputPort;
+    }
+
+    @Override
+    public void setup(PortContext context)
+    {
+      if (outputPort != null) {
+        outputPort.setup(context);
+      }
+    }
+
+    @Override
+    public void teardown()
+    {
+      if (outputPort != null) {
+        outputPort.teardown();
+      }
+    }
+
+    @Override
+    public void setSink(Sink<Object> s)
+    {
+      if (outputPort != null) {
+        outputPort.setSink(s);
+      }
+    }
+
+    @Override
+    public Unifier<T> getUnifier()
+    {
+      return outputPort == null ? null : outputPort.getUnifier();
+    }
+  }
 }
+

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 5a3e167..21039cc 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -37,12 +37,14 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Maps;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Sets;
 
 import com.datatorrent.api.*;
 import com.datatorrent.api.Attribute.AttributeMap;
 import com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap;
+import com.datatorrent.api.Module.ProxyInputPort;
+import com.datatorrent.api.Module.ProxyOutputPort;
 import com.datatorrent.api.Operator.InputPort;
 import com.datatorrent.api.Operator.OutputPort;
 import com.datatorrent.api.Operator.Unifier;
@@ -152,6 +154,7 @@ public class LogicalPlan implements Serializable, DAG
   private final Attribute.AttributeMap attributes = new DefaultAttributeMap();
   private transient int nodeIndex = 0; // used for cycle validation
   private transient Stack<OperatorMeta> stack = new Stack<OperatorMeta>(); // used for cycle validation
+  private transient Map<String, ArrayListMultimap<OutputPort<?>, InputPort<?>>> streamLinks = new HashMap<>();
 
   @Override
   public Attribute.AttributeMap getAttributes()
@@ -1197,6 +1200,7 @@ public class LogicalPlan implements Serializable, DAG
         subModuleMeta.setParent(this);
         subModuleMeta.flattenModule(dag, conf);
       }
+      dag.applyStreamLinks();
       parentDAG.addDAGToCurrentDAG(this);
     }
 
@@ -1300,13 +1304,52 @@ public class LogicalPlan implements Serializable, DAG
   public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T>... sinks)
   {
     StreamMeta s = addStream(id);
-    s.setSource(source);
-    for (Operator.InputPort<?> sink: sinks) {
-      s.addSink(sink);
+    id = s.id;
+    ArrayListMultimap<OutputPort<?>, InputPort<?>> streamMap = ArrayListMultimap.create();
+    if (!(source instanceof ProxyOutputPort)) {
+      s.setSource(source);
+    }
+    for (Operator.InputPort<?> sink : sinks) {
+      if (source instanceof ProxyOutputPort || sink instanceof ProxyInputPort) {
+        streamMap.put(source, sink);
+        streamLinks.put(id, streamMap);
+      } else {
+        if (s.getSource() == null) {
+          s.setSource(source);
+        }
+        s.addSink(sink);
+      }
     }
     return s;
   }
 
+  /**
+   * This will be called once the Logical Dag is expanded, and the proxy input and proxy output ports are populated with
+   * the actual ports that they refer to This method adds sources and sinks for the StreamMeta objects which were left
+   * empty in the addStream call.
+   */
+  public void applyStreamLinks()
+  {
+    for (String id : streamLinks.keySet()) {
+      StreamMeta s = getStream(id);
+      for (Map.Entry<Operator.OutputPort<?>, Operator.InputPort<?>> pair : streamLinks.get(id).entries()) {
+        if (s.getSource() == null) {
+          Operator.OutputPort<?> outputPort = pair.getKey();
+          while (outputPort instanceof ProxyOutputPort) {
+            outputPort = ((ProxyOutputPort<?>)outputPort).get();
+          }
+          s.setSource(outputPort);
+        }
+
+        Operator.InputPort<?> inputPort = pair.getValue();
+        while (inputPort instanceof ProxyInputPort) {
+          inputPort = ((ProxyInputPort<?>)inputPort).get();
+        }
+        s.addSink(inputPort);
+      }
+    }
+  }
+
   @SuppressWarnings({ "unchecked", "rawtypes" })
   private void addDAGToCurrentDAG(ModuleMeta moduleMeta)
   {

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 6dc4c0c..483576a 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -2128,6 +2128,7 @@ public class LogicalPlanConfiguration {
     for (ModuleMeta moduleMeta : dag.getAllModules()) {
       moduleMeta.flattenModule(dag, conf);
     }
+    dag.applyStreamLinks();
   }
 
   public static Properties readProperties(String filePath) throws IOException

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java
new file mode 100644
index 0000000..97c015e
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.Module;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+
+/**
+ * Unit tests for testing Dag expansion with modules and proxy port substitution
+ */
+public class ModuleAppTest
+{
+
+  /*
+   * Input Operator - 1
+   */
+  static class DummyInputOperator extends BaseOperator implements InputOperator
+  {
+
+    Random r = new Random();
+    public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<Integer>();
+
+    @Override
+    public void emitTuples()
+    {
+      output.emit(r.nextInt());
+    }
+  }
+
+  /*
+   * Input Operator - 1.1
+   */
+  static class DummyOperatorAfterInput extends BaseOperator
+  {
+
+    public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
+    {
+      @Override
+      public void process(Integer tuple)
+      {
+        output.emit(tuple);
+      }
+    };
+    public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<Integer>();
+  }
+
+  /*
+   * Operator - 2
+   */
+  static class DummyOperator extends BaseOperator
+  {
+    int prop;
+
+    public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
+    {
+      @Override
+      public void process(Integer tuple)
+      {
+        LOG.debug(tuple.intValue() + " processed");
+        output.emit(tuple);
+      }
+    };
+    public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<Integer>();
+  }
+
+  /*
+   * Output Operator - 3
+   */
+  static class DummyOutputOperator extends BaseOperator
+  {
+    int prop;
+
+    public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
+    {
+      @Override
+      public void process(Integer tuple)
+      {
+        LOG.debug(tuple.intValue() + " processed");
+      }
+    };
+  }
+
+  /*
+   * Module Definition
+   */
+  static class TestModule implements Module
+  {
+
+    public transient ProxyInputPort<Integer> moduleInput = new Module.ProxyInputPort<Integer>();
+    public transient ProxyOutputPort<Integer> moduleOutput = new Module.ProxyOutputPort<Integer>();
+
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      LOG.debug("Module - PopulateDAG");
+      DummyOperator dummyOperator = dag.addOperator("DummyOperator", new DummyOperator());
+      moduleInput.set(dummyOperator.input);
+      moduleOutput.set(dummyOperator.output);
+    }
+  }
+
+  static class Application implements StreamingApplication
+  {
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      LOG.debug("Application - PopulateDAG");
+      DummyInputOperator dummyInputOperator = dag.addOperator("DummyInputOperator", new DummyInputOperator());
+      DummyOperatorAfterInput dummyOperatorAfterInput = dag.addOperator("DummyOperatorAfterInput",
+          new DummyOperatorAfterInput());
+      Module m1 = dag.addModule("TestModule1", new TestModule());
+      Module m2 = dag.addModule("TestModule2", new TestModule());
+      DummyOutputOperator dummyOutputOperator = dag.addOperator("DummyOutputOperator", new DummyOutputOperator());
+      dag.addStream("Operator To Operator", dummyInputOperator.output, dummyOperatorAfterInput.input);
+      dag.addStream("Operator To Module", dummyOperatorAfterInput.output, ((TestModule)m1).moduleInput);
+      dag.addStream("Module To Module", ((TestModule)m1).moduleOutput, ((TestModule)m2).moduleInput);
+      dag.addStream("Module To Operator", ((TestModule)m2).moduleOutput, dummyOutputOperator.input);
+    }
+  }
+
+  @Test
+  public void validateTestApplication()
+  {
+    Configuration conf = new Configuration(false);
+    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+    LogicalPlan dag = new LogicalPlan();
+    lpc.prepareDAG(dag, new Application(), "TestApp");
+
+    Assert.assertEquals(2, dag.getAllModules().size(), 2);
+    Assert.assertEquals(5, dag.getAllOperators().size());
+    Assert.assertEquals(4, dag.getAllStreams().size());
+    dag.validate();
+  }
+
+  private static Logger LOG = LoggerFactory.getLogger(ModuleAppTest.class);
+}

http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java
new file mode 100644
index 0000000..5bfd8f1
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java
@@ -0,0 +1,552 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.Module;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+
+public class TestModuleExpansion
+{
+  static class DummyInputOperator extends BaseOperator implements InputOperator
+  {
+    private int inputOperatorProp = 0;
+
+    Random r = new Random();
+    public transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
+
+    @Override
+    public void emitTuples()
+    {
+      out.emit(r.nextInt());
+    }
+
+    public int getInputOperatorProp()
+    {
+      return inputOperatorProp;
+    }
+
+    public void setInputOperatorProp(int inputOperatorProp)
+    {
+      this.inputOperatorProp = inputOperatorProp;
+    }
+  }
+
+  static class DummyOperator extends BaseOperator
+  {
+    private int operatorProp = 0;
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient DefaultOutputPort<Integer> out1 = new DefaultOutputPort<>();
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient DefaultOutputPort<Integer> out2 = new DefaultOutputPort<>();
+
+    @InputPortFieldAnnotation(optional = true)
+    public final transient DefaultInputPort<Integer> in = new DefaultInputPort<Integer>()
+    {
+      @Override
+      public void process(Integer tuple)
+      {
+        out1.emit(tuple);
+        out2.emit(tuple);
+      }
+    };
+
+    public int getOperatorProp()
+    {
+      return operatorProp;
+    }
+
+    public void setOperatorProp(int operatorProp)
+    {
+      this.operatorProp = operatorProp;
+    }
+  }
+
+  static class Level1Module implements Module
+  {
+    private int level1ModuleProp = 0;
+
+    @InputPortFieldAnnotation(optional = true)
+    public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient ProxyOutputPort<Integer> mOut = new ProxyOutputPort<>();
+
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      DummyOperator o1 = dag.addOperator("O1", new DummyOperator());
+      o1.setOperatorProp(level1ModuleProp);
+      mIn.set(o1.in);
+      mOut.set(o1.out1);
+    }
+
+    public int getLevel1ModuleProp()
+    {
+      return level1ModuleProp;
+    }
+
+    public void setLevel1ModuleProp(int level1ModuleProp)
+    {
+      this.level1ModuleProp = level1ModuleProp;
+    }
+  }
+
+  static class Level2ModuleA implements Module
+  {
+    private int level2ModuleAProp1 = 0;
+    private int level2ModuleAProp2 = 0;
+    private int level2ModuleAProp3 = 0;
+
+    @InputPortFieldAnnotation(optional = true)
+    public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient ProxyOutputPort<Integer> mOut1 = new ProxyOutputPort<>();
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient ProxyOutputPort<Integer> mOut2 = new ProxyOutputPort<>();
+
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      Level1Module m1 = dag.addModule("M1", new Level1Module());
+      m1.setLevel1ModuleProp(level2ModuleAProp1);
+
+      Level1Module m2 = dag.addModule("M2", new Level1Module());
+      m2.setLevel1ModuleProp(level2ModuleAProp2);
+
+      DummyOperator o1 = dag.addOperator("O1", new DummyOperator());
+      o1.setOperatorProp(level2ModuleAProp3);
+
+      dag.addStream("M1_M2&O1", m1.mOut, m2.mIn, o1.in);
+
+      mIn.set(m1.mIn);
+      mOut1.set(m2.mOut);
+      mOut2.set(o1.out1);
+    }
+
+    public int getLevel2ModuleAProp1()
+    {
+      return level2ModuleAProp1;
+    }
+
+    public void setLevel2ModuleAProp1(int level2ModuleAProp1)
+    {
+      this.level2ModuleAProp1 = level2ModuleAProp1;
+    }
+
+    public int getLevel2ModuleAProp2()
+    {
+      return level2ModuleAProp2;
+    }
+
+    public void setLevel2ModuleAProp2(int level2ModuleAProp2)
+    {
+      this.level2ModuleAProp2 = level2ModuleAProp2;
+    }
+
+    public int getLevel2ModuleAProp3()
+    {
+      return level2ModuleAProp3;
+    }
+
+    public void setLevel2ModuleAProp3(int level2ModuleAProp3)
+    {
+      this.level2ModuleAProp3 = level2ModuleAProp3;
+    }
+  }
+
+  static class Level2ModuleB implements Module
+  {
+    private int level2ModuleBProp1 = 0;
+    private int level2ModuleBProp2 = 0;
+    private int level2ModuleBProp3 = 0;
+
+    @InputPortFieldAnnotation(optional = true)
+    public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient ProxyOutputPort<Integer> mOut1 = new ProxyOutputPort<>();
+
+    @OutputPortFieldAnnotation(optional = true)
+    public final transient ProxyOutputPort<Integer> mOut2 = new ProxyOutputPort<>();
+
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      DummyOperator o1 = dag.addOperator("O1", new DummyOperator());
+      o1.setOperatorProp(level2ModuleBProp1);
+
+      Level1Module m1 = dag.addModule("M1", new Level1Module());
+      m1.setLevel1ModuleProp(level2ModuleBProp2);
+
+      DummyOperator o2 = dag.addOperator("O2", new DummyOperator());
+      o2.setOperatorProp(level2ModuleBProp3);
+
+      dag.addStream("O1_M1", o1.out1, m1.mIn);
+      dag.addStream("O1_O2", o1.out2, o2.in);
+
+      mIn.set(o1.in);
+      mOut1.set(m1.mOut);
+      mOut2.set(o2.out1);
+    }
+
+    public int getLevel2ModuleBProp1()
+    {
+      return level2ModuleBProp1;
+    }
+
+    public void setLevel2ModuleBProp1(int level2ModuleBProp1)
+    {
+      this.level2ModuleBProp1 = level2ModuleBProp1;
+    }
+
+    public int getLevel2ModuleBProp2()
+    {
+      return level2ModuleBProp2;
+    }
+
+    public void setLevel2ModuleBProp2(int level2ModuleBProp2)
+    {
+      this.level2ModuleBProp2 = level2ModuleBProp2;
+    }
+
+    public int getLevel2ModuleBProp3()
+    {
+      return level2ModuleBProp3;
+    }
+
+    public void setLevel2ModuleBProp3(int level2ModuleBProp3)
+    {
+      this.level2ModuleBProp3 = level2ModuleBProp3;
+    }
+  }
+
+  static class Level3Module implements Module
+  {
+
+    public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+    public final transient ProxyOutputPort<Integer> mOut1 = new ProxyOutputPort<>();
+    public final transient ProxyOutputPort<Integer> mOut2 = new ProxyOutputPort<>();
+
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      DummyOperator op = dag.addOperator("O1", new DummyOperator());
+      Level2ModuleB m1 = dag.addModule("M1", new Level2ModuleB());
+      Level1Module m2 = dag.addModule("M2", new Level1Module());
+
+      dag.addStream("s1", op.out1, m1.mIn);
+      dag.addStream("s2", op.out2, m2.mIn);
+
+      mIn.set(op.in);
+      mOut1.set(m1.mOut1);
+      mOut2.set(m2.mOut);
+    }
+  }
+
+  static class NestedModuleApp implements StreamingApplication
+  {
+    @Override
+    public void populateDAG(DAG dag, Configuration conf)
+    {
+      DummyInputOperator o1 = dag.addOperator("O1", new DummyInputOperator());
+      o1.setInputOperatorProp(1);
+
+      DummyOperator o2 = dag.addOperator("O2", new DummyOperator());
+      o2.setOperatorProp(2);
+
+      Level2ModuleA ma = dag.addModule("Ma", new Level2ModuleA());
+      ma.setLevel2ModuleAProp1(11);
+      ma.setLevel2ModuleAProp2(12);
+      ma.setLevel2ModuleAProp3(13);
+
+      Level2ModuleB mb = dag.addModule("Mb", new Level2ModuleB());
+      mb.setLevel2ModuleBProp1(21);
+      mb.setLevel2ModuleBProp2(22);
+      mb.setLevel2ModuleBProp3(23);
+
+      Level2ModuleA mc = dag.addModule("Mc", new Level2ModuleA());
+      mc.setLevel2ModuleAProp1(31);
+      mc.setLevel2ModuleAProp2(32);
+      mc.setLevel2ModuleAProp3(33);
+
+      Level2ModuleB md = dag.addModule("Md", new Level2ModuleB());
+      md.setLevel2ModuleBProp1(41);
+      md.setLevel2ModuleBProp2(42);
+      md.setLevel2ModuleBProp3(43);
+
+      Level3Module me = dag.addModule("Me", new Level3Module());
+
+      dag.addStream("O1_O2", o1.out, o2.in, me.mIn);
+      dag.addStream("O2_Ma", o2.out1, ma.mIn);
+      dag.addStream("Ma_Mb", ma.mOut1, mb.mIn);
+      dag.addStream("Ma_Md", ma.mOut2, md.mIn);
+      dag.addStream("Mb_Mc", mb.mOut2, mc.mIn);
+    }
+  }
+
+  @Test
+  public void testModuleExtreme()
+  {
+    StreamingApplication app = new NestedModuleApp();
+    Configuration conf = new Configuration(false);
+    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+    LogicalPlan dag = new LogicalPlan();
+    lpc.prepareDAG(dag, app, "ModuleApp");
+
+    dag.validate();
+    validateTopLevelOperators(dag);
+    validateTopLevelStreams(dag);
+    validatePublicMethods(dag);
+  }
+
+  private void validateTopLevelStreams(LogicalPlan dag)
+  {
+    List<String> streamNames = new ArrayList<>();
+    for (LogicalPlan.StreamMeta streamMeta : dag.getAllStreams()) {
+      streamNames.add(streamMeta.getName());
+    }
+
+    Assert.assertTrue(streamNames.contains(componentName("Mb", "O1_M1")));
+    Assert.assertTrue(streamNames.contains("O2_Ma"));
+    Assert.assertTrue(streamNames.contains("Mb_Mc"));
+    Assert.assertTrue(streamNames.contains(componentName("Mb", "O1_O2")));
+    Assert.assertTrue(streamNames.contains(componentName("Ma", "M1_M2&O1")));
+    Assert.assertTrue(streamNames.contains(componentName("Md", "O1_M1")));
+    Assert.assertTrue(streamNames.contains(componentName("Ma_Md")));
+    Assert.assertTrue(streamNames.contains(componentName("Mc", "M1_M2&O1")));
+    Assert.assertTrue(streamNames.contains(componentName("Md", "O1_O2")));
+    Assert.assertTrue(streamNames.contains("Ma_Mb"));
+    Assert.assertTrue(streamNames.contains("O1_O2"));
+
+    validateSeperateStream(dag, componentName("Mb", "O1_M1"), componentName("Mb", "O1"),
+        componentName("Mb", "M1", "O1"));
+    validateSeperateStream(dag, "O2_Ma", "O2", componentName("Ma", "M1", "O1"));
+    validateSeperateStream(dag, "Mb_Mc", componentName("Mb", "O2"), componentName("Mc", "M1", "O1"));
+    validateSeperateStream(dag, componentName("Mb", "O1_O2"), componentName("Mb", "O1"), componentName("Mb", "O2"));
+    validateSeperateStream(dag, componentName("Ma", "M1_M2&O1"), componentName("Ma", "M1", "O1"),
+        componentName("Ma", "O1"), componentName("Ma", "M2", "O1"));
+    validateSeperateStream(dag, componentName("Md", "O1_M1"), componentName("Md", "O1"),
+        componentName("Md", "M1", "O1"));
+    validateSeperateStream(dag, "Ma_Md", componentName("Ma", "O1"), componentName("Md", "O1"));
+    validateSeperateStream(dag, componentName("Mc", "M1_M2&O1"), componentName("Mc", "M1", "O1"),
+        componentName("Mc", "O1"), componentName("Mc", "M2", "O1"));
+    validateSeperateStream(dag, componentName("Md", "O1_O2"), componentName("Md", "O1"), componentName("Md", "O2"));
+    validateSeperateStream(dag, "Ma_Mb", componentName("Ma", "M2", "O1"), componentName("Mb", "O1"));
+    validateSeperateStream(dag, "O1_O2", "O1", "O2", componentName("Me", "O1"));
+  }
+
+  private void validateSeperateStream(LogicalPlan dag, String streamName, String inputOperatorName,
+      String... outputOperatorNames)
+  {
+    LogicalPlan.StreamMeta streamMeta = dag.getStream(streamName);
+    String sourceName = streamMeta.getSource().getOperatorMeta().getName();
+
+    List<String> sinksName = new ArrayList<>();
+    for (LogicalPlan.InputPortMeta inputPortMeta : streamMeta.getSinks()) {
+      sinksName.add(inputPortMeta.getOperatorWrapper().getName());
+    }
+
+    Assert.assertTrue(inputOperatorName.equals(sourceName));
+    Assert.assertEquals(outputOperatorNames.length, sinksName.size());
+
+    for (String outputOperatorName : outputOperatorNames) {
+      Assert.assertTrue(sinksName.contains(outputOperatorName));
+    }
+  }
+
+  private void validateTopLevelOperators(LogicalPlan dag)
+  {
+    List<String> operatorNames = new ArrayList<>();
+    for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
+      operatorNames.add(operatorMeta.getName());
+    }
+    Assert.assertTrue(operatorNames.contains("O1"));
+    Assert.assertTrue(operatorNames.contains("O2"));
+    Assert.assertTrue(operatorNames.contains(componentName("Ma", "M1", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Ma", "M2", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Ma", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Mb", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Mb", "M1", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Mb", "O2")));
+    Assert.assertTrue(operatorNames.contains(componentName("Mc", "M1", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Mc", "M2", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Mc", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Md", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Md", "M1", "O1")));
+    Assert.assertTrue(operatorNames.contains(componentName("Md", "O2")));
+
+    validateOperatorPropertyValue(dag, "O1", 1);
+    validateOperatorPropertyValue(dag, "O2", 2);
+    validateOperatorPropertyValue(dag, componentName("Ma", "M1", "O1"), 11);
+    validateOperatorPropertyValue(dag, componentName("Ma", "M2", "O1"), 12);
+    validateOperatorPropertyValue(dag, componentName("Ma", "O1"), 13);
+    validateOperatorPropertyValue(dag, componentName("Mb", "O1"), 21);
+    validateOperatorPropertyValue(dag, componentName("Mb", "M1", "O1"), 22);
+    validateOperatorPropertyValue(dag, componentName("Mb", "O2"), 23);
+    validateOperatorPropertyValue(dag, componentName("Mc", "M1", "O1"), 31);
+    validateOperatorPropertyValue(dag, componentName("Mc", "M2", "O1"), 32);
+    validateOperatorPropertyValue(dag, componentName("Mc", "O1"), 33);
+    validateOperatorPropertyValue(dag, componentName("Md", "O1"), 41);
+    validateOperatorPropertyValue(dag, componentName("Md", "M1", "O1"), 42);
+    validateOperatorPropertyValue(dag, componentName("Md", "O2"), 43);
+
+    validateOperatorParent(dag, "O1", null);
+    validateOperatorParent(dag, "O2", null);
+    validateOperatorParent(dag, componentName("Ma", "M1", "O1"), componentName("Ma", "M1"));
+    validateOperatorParent(dag, componentName("Ma", "M2", "O1"), componentName("Ma", "M2"));
+    validateOperatorParent(dag, componentName("Ma", "O1"), "Ma");
+    validateOperatorParent(dag, componentName("Mb", "O1"), "Mb");
+    validateOperatorParent(dag, componentName("Mb", "M1", "O1"), componentName("Mb", "M1"));
+    validateOperatorParent(dag, componentName("Mb", "O2"), "Mb");
+    validateOperatorParent(dag, componentName("Mc", "M1", "O1"), componentName("Mc", "M1"));
+    validateOperatorParent(dag, componentName("Mc", "M2", "O1"), componentName("Mc", "M2"));
+    validateOperatorParent(dag, componentName("Mc", "O1"), "Mc");
+    validateOperatorParent(dag, componentName("Md", "O1"), "Md");
+    validateOperatorParent(dag, componentName("Md", "M1", "O1"), componentName("Md", "M1"));
+    validateOperatorParent(dag, componentName("Md", "O2"), "Md");
+  }
+
+  private void validateOperatorParent(LogicalPlan dag, String operatorName, String parentModuleName)
+  {
+    LogicalPlan.OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
+    if (parentModuleName == null) {
+      Assert.assertNull(operatorMeta.getModuleName());
+    } else {
+      Assert.assertTrue(parentModuleName.equals(operatorMeta.getModuleName()));
+    }
+  }
+
+  private void validateOperatorPropertyValue(LogicalPlan dag, String operatorName, int expectedValue)
+  {
+    LogicalPlan.OperatorMeta oMeta = dag.getOperatorMeta(operatorName);
+    if (operatorName.equals("O1")) {
+      DummyInputOperator operator = (DummyInputOperator)oMeta.getOperator();
+      Assert.assertEquals(expectedValue, operator.getInputOperatorProp());
+    } else {
+      DummyOperator operator = (DummyOperator)oMeta.getOperator();
+      Assert.assertEquals(expectedValue, operator.getOperatorProp());
+    }
+  }
+
+  private void validatePublicMethods(LogicalPlan dag)
+  {
+    // Logical dag contains 4 modules added on top level.
+    List<String> moduleNames = new ArrayList<>();
+    for (LogicalPlan.ModuleMeta moduleMeta : dag.getAllModules()) {
+      moduleNames.add(moduleMeta.getName());
+    }
+    Assert.assertTrue(moduleNames.contains("Ma"));
+    Assert.assertTrue(moduleNames.contains("Mb"));
+    Assert.assertTrue(moduleNames.contains("Mc"));
+    Assert.assertTrue(moduleNames.contains("Md"));
+    Assert.assertTrue(moduleNames.contains("Me"));
+    Assert.assertEquals("Number of modules are 5", 5, dag.getAllModules().size());
+
+    // correct module meta is returned by getMeta call.
+    LogicalPlan.ModuleMeta m = dag.getModuleMeta("Ma");
+    Assert.assertEquals("Name of module is Ma", m.getName(), "Ma");
+
+  }
+
+  private static String componentName(String... names)
+  {
+    if (names.length == 0) {
+      return "";
+    }
+    StringBuilder sb = new StringBuilder(names[0]);
+    for (int i = 1; i < names.length; i++) {
+      sb.append(LogicalPlan.MODULE_NAMESPACE_SEPARATOR);
+      sb.append(names[i]);
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Generate a conflict, Add a top level operator with name "m1_O1",
+   * and add a module "m1" which will populate operator "O1", causing name conflict with
+   * top level operator.
+   */
+  @Test(expected = java.lang.IllegalArgumentException.class)
+  public void conflictingNamesWithExpandedModule()
+  {
+    Configuration conf = new Configuration(false);
+    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+    LogicalPlan dag = new LogicalPlan();
+    DummyInputOperator in = dag.addOperator(componentName("m1", "O1"), new DummyInputOperator());
+    Level2ModuleA module = dag.addModule("m1", new Level2ModuleA());
+    dag.addStream("s1", in.out, module.mIn);
+    lpc.prepareDAG(dag, null, "ModuleApp");
+    dag.validate();
+  }
+
+  /**
+   * Module and Operator with same name is not allowed in a DAG, to prevent properties
+   * conflict.
+   */
+  @Test(expected = java.lang.IllegalArgumentException.class)
+  public void conflictingNamesWithOperator1()
+  {
+    Configuration conf = new Configuration(false);
+    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+    LogicalPlan dag = new LogicalPlan();
+    DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
+    Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
+    dag.addStream("s1", in.out, module.mIn);
+    lpc.prepareDAG(dag, null, "ModuleApp");
+    dag.validate();
+  }
+
+  /**
+   * Module and Operator with same name is not allowed in a DAG, to prevent properties
+   * conflict.
+   */
+  @Test(expected = java.lang.IllegalArgumentException.class)
+  public void conflictingNamesWithOperator2()
+  {
+    Configuration conf = new Configuration(false);
+    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+    LogicalPlan dag = new LogicalPlan();
+    Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
+    DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
+    dag.addStream("s1", in.out, module.mIn);
+    lpc.prepareDAG(dag, null, "ModuleApp");
+    dag.validate();
+  }
+}