You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2015/12/22 08:01:19 UTC
[1/5] incubator-apex-core git commit: APEX-103 - Add module and dag
interface in API
Repository: incubator-apex-core
Updated Branches:
refs/heads/devel-3 0d5bfa5be -> 47b3ce8e4
APEX-103 - Add module and dag interface in API
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/2f1e1dfe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/2f1e1dfe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/2f1e1dfe
Branch: refs/heads/devel-3
Commit: 2f1e1dfeb400bd7f01275a95522766abe909d936
Parents: 0d5bfa5
Author: Vlad Rozov <v....@datatorrent.com>
Authored: Wed Sep 23 20:30:51 2015 -0700
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Mon Dec 21 23:41:04 2015 +0530
----------------------------------------------------------------------
api/src/main/java/com/datatorrent/api/DAG.java | 24 +++++++++++++++++
.../main/java/com/datatorrent/api/Module.java | 28 ++++++++++++++++++++
.../stram/plan/logical/LogicalPlan.java | 24 +++++++++++++++++
3 files changed, 76 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/2f1e1dfe/api/src/main/java/com/datatorrent/api/DAG.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/DAG.java b/api/src/main/java/com/datatorrent/api/DAG.java
index 6b1d1b2..abe2954 100644
--- a/api/src/main/java/com/datatorrent/api/DAG.java
+++ b/api/src/main/java/com/datatorrent/api/DAG.java
@@ -20,6 +20,8 @@ package com.datatorrent.api;
import java.io.Serializable;
+import org.apache.hadoop.classification.InterfaceStability;
+
import com.datatorrent.api.Context.DAGContext;
/**
@@ -157,6 +159,16 @@ public interface DAG extends DAGContext, Serializable
public OutputPortMeta getMeta(Operator.OutputPort<?> port);
}
+ @InterfaceStability.Evolving
+ interface ModuleMeta extends Serializable, Context
+ {
+ String getName();
+
+ InputPortMeta getMeta(Operator.InputPort<?> port);
+
+ OutputPortMeta getMeta(Operator.OutputPort<?> port);
+ }
+
/**
* Add new instance of operator under given name to the DAG.
* The operator class must have a default constructor.
@@ -179,6 +191,12 @@ public interface DAG extends DAGContext, Serializable
*/
public abstract <T extends Operator> T addOperator(String name, T operator);
+ @InterfaceStability.Evolving
+ <T extends Module> T addModule(String name, Class<T> moduleClass);
+
+ @InterfaceStability.Evolving
+ <T extends Module> T addModule(String name, T module);
+
/**
* <p>addStream.</p>
* @param id Identifier of the stream that will be used to identify stream in DAG
@@ -256,9 +274,15 @@ public interface DAG extends DAGContext, Serializable
*/
public abstract OperatorMeta getOperatorMeta(String operatorId);
+ @InterfaceStability.Evolving
+ ModuleMeta getModuleMeta(String moduleId);
+
/**
* <p>getMeta.</p>
*/
public abstract OperatorMeta getMeta(Operator operator);
+ @InterfaceStability.Evolving
+ ModuleMeta getMeta(Module module);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/2f1e1dfe/api/src/main/java/com/datatorrent/api/Module.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Module.java b/api/src/main/java/com/datatorrent/api/Module.java
new file mode 100644
index 0000000..1220fc1
--- /dev/null
+++ b/api/src/main/java/com/datatorrent/api/Module.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.api;
+
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+@InterfaceStability.Evolving
+public interface Module
+{
+ void populateDAG(DAG dag, Configuration conf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/2f1e1dfe/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 6405644..cca45d8 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -1079,6 +1079,18 @@ public class LogicalPlan implements Serializable, DAG
return operator;
}
+ @Override
+ public <T extends Module> T addModule(String name, Class<T> moduleClass)
+ {
+ throw new UnsupportedOperationException("Modules are not supported");
+ }
+
+ @Override
+ public <T extends Module> T addModule(String name, T module)
+ {
+ throw new UnsupportedOperationException("Modules are not supported");
+ }
+
public void removeOperator(Operator operator)
{
OperatorMeta om = getMeta(operator);
@@ -1231,6 +1243,12 @@ public class LogicalPlan implements Serializable, DAG
}
@Override
+ public ModuleMeta getModuleMeta(String moduleName)
+ {
+ throw new UnsupportedOperationException("Modules are not supported");
+ }
+
+ @Override
public OperatorMeta getMeta(Operator operator)
{
// TODO: cache mapping
@@ -1242,6 +1260,12 @@ public class LogicalPlan implements Serializable, DAG
throw new IllegalArgumentException("Operator not associated with the DAG: " + operator);
}
+ @Override
+ public ModuleMeta getMeta(Module module)
+ {
+ throw new UnsupportedOperationException("Modules are not supported");
+ }
+
public int getMaxContainerCount()
{
return this.getValue(CONTAINERS_MAX_COUNT);
[2/5] incubator-apex-core git commit: APEXCORE-105 Introduce module
meta Inject properties through xml file on modules.
Posted by th...@apache.org.
APEXCORE-105 Introduce module meta
Inject properties through xml file on modules.
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/b0360d45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/b0360d45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/b0360d45
Branch: refs/heads/devel-3
Commit: b0360d45a361e462124db9c3000977987ca830e6
Parents: 2f1e1df
Author: Tushar R. Gosavi <tu...@apache.org>
Authored: Tue Oct 6 13:48:53 2015 +0530
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Tue Dec 22 01:16:51 2015 +0530
----------------------------------------------------------------------
api/src/main/java/com/datatorrent/api/DAG.java | 2 +
.../stram/codec/LogicalPlanSerializer.java | 2 +-
.../stram/plan/logical/LogicalPlan.java | 120 ++++++++++-
.../plan/logical/LogicalPlanConfiguration.java | 99 +++++++--
.../stram/webapp/StramWebServices.java | 2 +-
.../logical/module/TestModuleProperties.java | 58 +++++
.../stram/plan/logical/module/TestModules.java | 216 +++++++++++++++++++
7 files changed, 472 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/api/src/main/java/com/datatorrent/api/DAG.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/DAG.java b/api/src/main/java/com/datatorrent/api/DAG.java
index abe2954..1dce402 100644
--- a/api/src/main/java/com/datatorrent/api/DAG.java
+++ b/api/src/main/java/com/datatorrent/api/DAG.java
@@ -164,6 +164,8 @@ public interface DAG extends DAGContext, Serializable
{
String getName();
+ Module getModule();
+
InputPortMeta getMeta(Operator.InputPort<?> port);
OutputPortMeta getMeta(Operator.OutputPort<?> port);
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
index 9e5ac04..90dd2b5 100644
--- a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
+++ b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
@@ -212,7 +212,7 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
Operator operator = operatorMeta.getOperator();
props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
- BeanMap operatorProperties = LogicalPlanConfiguration.getOperatorProperties(operator);
+ BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
@SuppressWarnings("rawtypes")
Iterator entryIterator = operatorProperties.entryIterator();
while (entryIterator.hasNext()) {
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index cca45d8..53e81bc 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.datatorrent.api.*;
@@ -48,7 +49,6 @@ import com.datatorrent.api.Operator.Unifier;
import com.datatorrent.api.annotation.InputPortFieldAnnotation;
import com.datatorrent.api.annotation.OperatorAnnotation;
import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-
import com.datatorrent.common.experimental.AppData;
import com.datatorrent.common.metric.MetricsAggregator;
import com.datatorrent.common.metric.SingleMetricAggregator;
@@ -1067,7 +1067,7 @@ public class LogicalPlan implements Serializable, DAG
public <T extends Operator> T addOperator(String name, T operator)
{
if (operators.containsKey(name)) {
- if (operators.get(name) == (Object)operator) {
+ if (operators.get(name).operator == operator) {
return operator;
}
throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
@@ -1079,16 +1079,109 @@ public class LogicalPlan implements Serializable, DAG
return operator;
}
- @Override
- public <T extends Module> T addModule(String name, Class<T> moduleClass)
+ public final class ModuleMeta implements DAG.ModuleMeta, Serializable
{
- throw new UnsupportedOperationException("Modules are not supported");
+ private final LinkedHashMap<InputPortMeta, StreamMeta> inputStreams = new LinkedHashMap<>();
+ private final LinkedHashMap<OutputPortMeta, StreamMeta> outputStreams = new LinkedHashMap<>();
+ private final Attribute.AttributeMap attributes;
+ @SuppressWarnings("unused")
+ private final int id;
+ @NotNull
+ private final String name;
+ private transient Integer nindex; // for cycle detection
+ private transient Integer lowlink; // for cycle detection
+ private transient Module module;
+
+ public ModuleMeta(String name, Module module)
+ {
+ this(name, module, new DefaultAttributeMap());
+ }
+
+ public ModuleMeta(String name, Module module, DefaultAttributeMap attributeMap)
+ {
+ LOG.debug("Initializing {} as {}", name, module.getClass().getName());
+ this.name = name;
+ this.module = module;
+ this.id = logicalOperatorSequencer.decrementAndGet();
+ this.attributes = attributeMap;
+ }
+
+ @Override
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public Module getModule()
+ {
+ return module;
+ }
+
+ @Override
+ public DAG.InputPortMeta getMeta(InputPort<?> port)
+ {
+ return null;
+ }
+
+ @Override
+ public DAG.OutputPortMeta getMeta(OutputPort<?> port)
+ {
+ return null;
+ }
+
+ @Override
+ public Attribute.AttributeMap getAttributes()
+ {
+ return null;
+ }
+
+ @Override
+ public <T> T getValue(Attribute<T> key)
+ {
+ return null;
+ }
+
+ @Override
+ public void setCounters(Object counters)
+ {
+
+ }
+
+ @Override
+ public void sendMetrics(Collection<String> metricNames)
+ {
+
+ }
}
+ public transient Map<String, ModuleMeta> modules = Maps.newHashMap();
+
@Override
public <T extends Module> T addModule(String name, T module)
{
- throw new UnsupportedOperationException("Modules are not supported");
+ if (modules.containsKey(name)) {
+ if (modules.get(name).module == module) {
+ return module;
+ }
+ throw new IllegalArgumentException("duplicate module is: " + modules.get(name));
+ }
+ ModuleMeta meta = new ModuleMeta(name, module);
+ modules.put(name, meta);
+ return module;
+ }
+
+ @Override
+ public <T extends Module> T addModule(String name, Class<T> clazz)
+ {
+ T instance;
+ try {
+ instance = clazz.newInstance();
+ } catch (Exception ex) {
+ throw new IllegalArgumentException(ex);
+ }
+ addModule(name, instance);
+ return instance;
}
public void removeOperator(Operator operator)
@@ -1231,6 +1324,10 @@ public class LogicalPlan implements Serializable, DAG
return Collections.unmodifiableCollection(this.operators.values());
}
+ public Collection<ModuleMeta> getAllModules() {
+ return Collections.unmodifiableCollection(this.modules.values());
+ }
+
public Collection<StreamMeta> getAllStreams()
{
return Collections.unmodifiableCollection(this.streams.values());
@@ -1242,10 +1339,9 @@ public class LogicalPlan implements Serializable, DAG
return this.operators.get(operatorName);
}
- @Override
public ModuleMeta getModuleMeta(String moduleName)
{
- throw new UnsupportedOperationException("Modules are not supported");
+ return null;
}
@Override
@@ -1260,10 +1356,14 @@ public class LogicalPlan implements Serializable, DAG
throw new IllegalArgumentException("Operator not associated with the DAG: " + operator);
}
- @Override
public ModuleMeta getMeta(Module module)
{
- throw new UnsupportedOperationException("Modules are not supported");
+ for (ModuleMeta m : getAllModules()) {
+ if (m.module == module) {
+ return m;
+ }
+ }
+ throw new IllegalArgumentException("Module not associated with the DAG: " + module);
}
public int getMaxContainerCount()
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 60bbdbe..9bbe85c 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -18,7 +18,6 @@
*/
package com.datatorrent.stram.plan.logical;
-
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -32,7 +31,6 @@ import java.lang.reflect.Type;
import java.util.*;
import java.util.Map.Entry;
-
import javax.validation.ValidationException;
import com.google.common.annotations.VisibleForTesting;
@@ -47,6 +45,7 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.collections.CollectionUtils;
@@ -61,10 +60,10 @@ import com.datatorrent.api.Context.DAGContext;
import com.datatorrent.api.Context.OperatorContext;
import com.datatorrent.api.Context.PortContext;
import com.datatorrent.api.annotation.ApplicationAnnotation;
-
import com.datatorrent.stram.StramUtils;
import com.datatorrent.stram.client.StramClientUtils;
import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.ModuleMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
@@ -2114,6 +2113,10 @@ public class LogicalPlanConfiguration {
if (dag.getAttributes().get(Context.DAGContext.APPLICATION_NAME) == null) {
dag.setAttribute(Context.DAGContext.APPLICATION_NAME, appName);
}
+
+ // Expand the modules within the dag recursively
+ setModuleProperties(dag, appName);
+
// inject external operator configuration
setOperatorConfiguration(dag, appConfs, appName);
setStreamConfiguration(dag, appConfs, appName);
@@ -2138,7 +2141,7 @@ public class LogicalPlanConfiguration {
public Map<String, String> getProperties(OperatorMeta ow, String appName) {
List<AppConf> appConfs = stramConf.getMatchingChildConf(appName, StramElement.APPLICATION);
List<OperatorConf> opConfs = getMatchingChildConf(appConfs, ow.getName(), StramElement.OPERATOR);
- return getProperties(ow, opConfs, appName);
+ return getProperties(getPropertyArgs(ow), opConfs, appName);
}
private Map<String,String> getApplicationProperties(List<AppConf> appConfs){
@@ -2153,17 +2156,17 @@ public class LogicalPlanConfiguration {
/**
* Get the configuration opProps for the given operator.
* These can be operator specific settings or settings from matching templates.
- * @param ow
+ * @param pa
* @param opConfs
* @param appName
*/
- private Map<String, String> getProperties(OperatorMeta ow, List<OperatorConf> opConfs, String appName)
+ private Map<String, String> getProperties(PropertyArgs pa, List<OperatorConf> opConfs, String appName)
{
Map<String, String> opProps = Maps.newHashMap();
Map<String, TemplateConf> templates = stramConf.getChildren(StramElement.TEMPLATE);
// list of all templates that match operator, ordered by priority
if (!templates.isEmpty()) {
- TreeMap<Integer, TemplateConf> matchingTemplates = getMatchingTemplates(ow, appName, templates);
+ TreeMap<Integer, TemplateConf> matchingTemplates = getMatchingTemplates(pa, appName, templates);
if (matchingTemplates != null && !matchingTemplates.isEmpty()) {
// combined map of prioritized template settings
for (TemplateConf t : matchingTemplates.descendingMap().values()) {
@@ -2197,23 +2200,46 @@ public class LogicalPlanConfiguration {
return refTemplates;
}
+ private static class PropertyArgs
+ {
+ String name;
+ String className;
+
+ public PropertyArgs(String name, String className)
+ {
+ this.name = name;
+ this.className = className;
+ }
+ }
+
+ private PropertyArgs getPropertyArgs(OperatorMeta om)
+ {
+ return new PropertyArgs(om.getName(), om.getOperator().getClass().getName());
+ }
+
+ private PropertyArgs getPropertyArgs(ModuleMeta mm)
+ {
+ return new PropertyArgs(mm.getName(), mm.getModule().getClass().getName());
+ }
+
/**
* Produce the collections of templates that apply for the given id.
- * @param ow
+ * @param pa
* @param appName
* @param templates
* @return TreeMap<Integer, TemplateConf>
*/
- private TreeMap<Integer, TemplateConf> getMatchingTemplates(OperatorMeta ow, String appName, Map<String, TemplateConf> templates) {
+ private TreeMap<Integer, TemplateConf> getMatchingTemplates(PropertyArgs pa, String appName, Map<String, TemplateConf> templates)
+ {
TreeMap<Integer, TemplateConf> tm = Maps.newTreeMap();
for (TemplateConf t : templates.values()) {
- if ((t.idRegExp != null && ow.getName().matches(t.idRegExp))) {
+ if ((t.idRegExp != null && pa.name.matches(t.idRegExp))) {
tm.put(1, t);
} else if (appName != null && t.appNameRegExp != null
&& appName.matches(t.appNameRegExp)) {
tm.put(2, t);
} else if (t.classNameRegExp != null
- && ow.getOperator().getClass().getName().matches(t.classNameRegExp)) {
+ && pa.className.matches(t.classNameRegExp)) {
tm.put(3, t);
}
}
@@ -2238,6 +2264,26 @@ public class LogicalPlanConfiguration {
}
}
+ /**
+ * Generic helper function to inject properties on the object.
+ *
+ * @param obj
+ * @param properties
+ * @param <T>
+ * @return
+ */
+ public static <T> T setObjectProperties(T obj, Map<String, String> properties)
+ {
+ try {
+ BeanUtils.populate(obj, properties);
+ return obj;
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException("Error setting operator properties", e);
+ } catch (InvocationTargetException e) {
+ throw new IllegalArgumentException("Error setting operator properties", e);
+ }
+ }
+
public static StreamingApplication setApplicationProperties(StreamingApplication application, Map<String, String> properties)
{
try {
@@ -2249,9 +2295,9 @@ public class LogicalPlanConfiguration {
}
}
- public static BeanMap getOperatorProperties(Operator operator)
+ public static BeanMap getObjectProperties(Object obj)
{
- return new BeanMap(operator);
+ return new BeanMap(obj);
}
/**
@@ -2266,12 +2312,26 @@ public class LogicalPlanConfiguration {
List<AppConf> appConfs = stramConf.getMatchingChildConf(applicationName, StramElement.APPLICATION);
for (OperatorMeta ow : dag.getAllOperators()) {
List<OperatorConf> opConfs = getMatchingChildConf(appConfs, ow.getName(), StramElement.OPERATOR);
- Map<String, String> opProps = getProperties(ow, opConfs, applicationName);
+ Map<String, String> opProps = getProperties(getPropertyArgs(ow), opConfs, applicationName);
setOperatorProperties(ow.getOperator(), opProps);
}
}
/**
+ * Set any properties from configuration on the modules in the DAG. This
+ * method may throw unchecked exception if the configuration contains
+ * properties that are invalid for a module.
+ *
+ * @param dag
+ * @param applicationName
+ */
+ public void setModuleProperties(LogicalPlan dag, String applicationName)
+ {
+ List<AppConf> appConfs = stramConf.getMatchingChildConf(applicationName, StramElement.APPLICATION);
+ setModuleConfiguration(dag, appConfs, applicationName);
+ }
+
+ /**
* Set the application configuration.
* @param dag
* @param appName
@@ -2298,7 +2358,7 @@ public class LogicalPlanConfiguration {
// Set the operator attributes
setAttributes(opConfs, ow.getAttributes());
// Set the operator opProps
- Map<String, String> opProps = getProperties(ow, opConfs, appName);
+ Map<String, String> opProps = getProperties(getPropertyArgs(ow), opConfs, appName);
setOperatorProperties(ow.getOperator(), opProps);
// Set the port attributes
@@ -2327,6 +2387,15 @@ public class LogicalPlanConfiguration {
}
}
+ private void setModuleConfiguration(final LogicalPlan dag, List<AppConf> appConfs, String appName)
+ {
+ for (final ModuleMeta mw : dag.getAllModules()) {
+ List<OperatorConf> opConfs = getMatchingChildConf(appConfs, mw.getName(), StramElement.OPERATOR);
+ Map<String, String> opProps = getProperties(getPropertyArgs(mw), opConfs, appName);
+ setObjectProperties(mw.getModule(), opProps);
+ }
+ }
+
private void setStreamConfiguration(LogicalPlan dag, List<AppConf> appConfs, String appAlias) {
for (StreamMeta sm : dag.getAllStreams()) {
List<StreamConf> smConfs = getMatchingChildConf(appConfs, sm.getName(), StramElement.STREAM);
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
index de085cd..6fdba00 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
@@ -715,7 +715,7 @@ public class StramWebServices
throw new NotFoundException();
}
- BeanMap operatorProperties = LogicalPlanConfiguration.getOperatorProperties(logicalOperator.getOperator());
+ BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalOperator.getOperator());
Map<String, Object> m = new HashMap<String, Object>();
@SuppressWarnings("rawtypes")
Iterator entryIterator = operatorProperties.entryIterator();
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java
new file mode 100644
index 0000000..7951e26
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleProperties.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+
+public class TestModuleProperties
+{
+ @Test
+ public void testModuleProperties()
+ {
+ Configuration conf = new Configuration(false);
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.myStringProperty", "myStringPropertyValue");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.stringArrayField", "a,b,c");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty.key1", "key1Val");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key1.dot)", "key1dotVal");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key2.dot)", "key2dotVal");
+
+ LogicalPlan dag = new LogicalPlan();
+ TestModules.GenericModule o1 = dag.addModule("o1", new TestModules.GenericModule());
+ TestModules.ValidationTestModule o2 = dag.addModule("o2", new TestModules.ValidationTestModule());
+
+ LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
+
+ pb.setModuleProperties(dag, "testSetOperatorProperties");
+ Assert.assertEquals("o1.myStringProperty", "myStringPropertyValue", o1.getMyStringProperty());
+ Assert.assertArrayEquals("o2.stringArrayField", new String[]{"a", "b", "c"}, o2.getStringArrayField());
+
+ Assert.assertEquals("o2.mapProperty.key1", "key1Val", o2.getMapProperty().get("key1"));
+ Assert.assertEquals("o2.mapProperty(key1.dot)", "key1dotVal", o2.getMapProperty().get("key1.dot"));
+ Assert.assertEquals("o2.mapProperty(key2.dot)", "key2dotVal", o2.getMapProperty().get("key2.dot"));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/b0360d45/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java
new file mode 100644
index 0000000..8fad613
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModules.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import java.util.Map;
+
+import javax.validation.Valid;
+import javax.validation.constraints.AssertTrue;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Module;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.stram.engine.GenericOperatorProperty;
+
+public class TestModules
+{
+
+ public static class GenericModule implements Module
+ {
+ private static final Logger LOG = LoggerFactory.getLogger(TestModules.class);
+
+ public volatile Object inport1Tuple = null;
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Object> outport2 = new DefaultOutputPort<Object>();
+
+ private String emitFormat;
+
+ public boolean booleanProperty;
+
+ private String myStringProperty;
+
+ private transient GenericOperatorProperty genericOperatorProperty = new GenericOperatorProperty("test");
+
+ public String getMyStringProperty()
+ {
+ return myStringProperty;
+ }
+
+ public void setMyStringProperty(String myStringProperty)
+ {
+ this.myStringProperty = myStringProperty;
+ }
+
+ public boolean isBooleanProperty()
+ {
+ return booleanProperty;
+ }
+
+ public void setBooleanProperty(boolean booleanProperty)
+ {
+ this.booleanProperty = booleanProperty;
+ }
+
+ public String propertySetterOnly;
+
+ /**
+ * setter w/o getter defined
+ *
+ * @param v
+ */
+ public void setStringPropertySetterOnly(String v)
+ {
+ this.propertySetterOnly = v;
+ }
+
+ public String getEmitFormat()
+ {
+ return emitFormat;
+ }
+
+ public void setEmitFormat(String emitFormat)
+ {
+ this.emitFormat = emitFormat;
+ }
+
+ public GenericOperatorProperty getGenericOperatorProperty()
+ {
+ return genericOperatorProperty;
+ }
+
+ public void setGenericOperatorProperty(GenericOperatorProperty genericOperatorProperty)
+ {
+ this.genericOperatorProperty = genericOperatorProperty;
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ LOG.debug("populateDAG of module called");
+ }
+ }
+
+ public static class ValidationTestModule implements Module
+ {
+ @NotNull
+ @Pattern(regexp = ".*malhar.*", message = "Value has to contain 'malhar'!")
+ private String stringField1;
+
+ @Min(2)
+ private int intField1;
+
+ @AssertTrue(message = "stringField1 should end with intField1")
+ private boolean isValidConfiguration()
+ {
+ return stringField1.endsWith(String.valueOf(intField1));
+ }
+
+ private String getterProperty2 = "";
+
+ @NotNull
+ public String getProperty2()
+ {
+ return getterProperty2;
+ }
+
+ public void setProperty2(String s)
+ {
+ // annotations need to be on the getter
+ getterProperty2 = s;
+ }
+
+ private String[] stringArrayField;
+
+ public String[] getStringArrayField()
+ {
+ return stringArrayField;
+ }
+
+ public void setStringArrayField(String[] stringArrayField)
+ {
+ this.stringArrayField = stringArrayField;
+ }
+
+ public class Nested
+ {
+ @NotNull
+ private String property = "";
+
+ public String getProperty()
+ {
+ return property;
+ }
+
+ public void setProperty(String property)
+ {
+ this.property = property;
+ }
+
+ }
+
+ @Valid
+ private final Nested nestedBean = new Nested();
+
+ private String stringProperty2;
+
+ public String getStringProperty2()
+ {
+ return stringProperty2;
+ }
+
+ public void setStringProperty2(String stringProperty2)
+ {
+ this.stringProperty2 = stringProperty2;
+ }
+
+ private Map<String, String> mapProperty = Maps.newHashMap();
+
+ public Map<String, String> getMapProperty()
+ {
+ return mapProperty;
+ }
+
+ public void setMapProperty(Map<String, String> mapProperty)
+ {
+ this.mapProperty = mapProperty;
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ }
+ }
+
+}
[5/5] incubator-apex-core git commit: APEXCORE-144,
APEXCORE-145 Rest api changes to view module information
Posted by th...@apache.org.
APEXCORE-144, APEXCORE-145 Rest api changes to view module information
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/47b3ce8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/47b3ce8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/47b3ce8e
Branch: refs/heads/devel-3
Commit: 47b3ce8e4019f325c2e91968cb120763f18bb174
Parents: c1314ea
Author: shubham <sh...@github.com>
Authored: Tue Nov 17 12:10:08 2015 +0530
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Tue Dec 22 12:03:55 2015 +0530
----------------------------------------------------------------------
.../stram/StreamingContainerManager.java | 20 +++
.../java/com/datatorrent/stram/cli/DTCli.java | 8 +-
.../stram/codec/LogicalPlanSerializer.java | 47 ++++++-
.../stram/plan/logical/LogicalPlan.java | 7 +-
.../plan/logical/LogicalPlanConfiguration.java | 4 +-
.../stram/webapp/StramWebServices.java | 123 +++++++++++++------
6 files changed, 158 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
index 29c6a2c..162245b 100644
--- a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
+++ b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
@@ -95,6 +95,7 @@ import com.datatorrent.stram.engine.WindowGenerator;
import com.datatorrent.stram.plan.logical.LogicalOperatorStatus;
import com.datatorrent.stram.plan.logical.LogicalPlan;
import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.ModuleMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
@@ -2246,6 +2247,25 @@ public class StreamingContainerManager implements PlanContext
return fillLogicalOperatorInfo(operatorMeta);
}
+ public ModuleMeta getModuleMeta(String moduleName)
+ {
+ return getModuleMeta(moduleName, getLogicalPlan());
+ }
+
+ private ModuleMeta getModuleMeta(String moduleName, LogicalPlan dag)
+ {
+ for (ModuleMeta m : dag.getAllModules()) {
+ if (m.getFullName().equals(moduleName)) {
+ return m;
+ }
+ ModuleMeta res = getModuleMeta(moduleName, m.getDag());
+ if (res != null) {
+ return res;
+ }
+ }
+ return null;
+ }
+
public List<LogicalOperatorInfo> getLogicalOperatorInfoList()
{
List<LogicalOperatorInfo> infoList = new ArrayList<LogicalOperatorInfo>();
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java b/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
index deb0967..696f497 100644
--- a/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
+++ b/engine/src/main/java/com/datatorrent/stram/cli/DTCli.java
@@ -2824,7 +2824,7 @@ public class DTCli
}
LogicalPlan logicalPlan = appFactory.createApp(submitApp.getLogicalPlanConfiguration());
map.put("applicationName", appFactory.getName());
- map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan));
+ map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan, false));
} finally {
if (raw) {
System.setOut(originalStream);
@@ -2840,7 +2840,7 @@ public class DTCli
LogicalPlan logicalPlan = appFactory.createApp(submitApp.getLogicalPlanConfiguration());
Map<String, Object> map = new HashMap<String, Object>();
map.put("applicationName", appFactory.getName());
- map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan));
+ map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan, false));
printJson(map);
} else if (filename.endsWith(".properties")) {
File file = new File(filename);
@@ -2849,7 +2849,7 @@ public class DTCli
LogicalPlan logicalPlan = appFactory.createApp(submitApp.getLogicalPlanConfiguration());
Map<String, Object> map = new HashMap<String, Object>();
map.put("applicationName", appFactory.getName());
- map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan));
+ map.put("logicalPlan", LogicalPlanSerializer.convertToMap(logicalPlan, false));
printJson(map);
} else {
StramAppLauncher submitApp = getStramAppLauncher(filename, config, commandLineInfo.ignorePom);
@@ -2893,7 +2893,7 @@ public class DTCli
Map<String, Object> map = new HashMap<String, Object>();
map.put("applicationName", appInfo.name);
if (appInfo.dag != null) {
- map.put("logicalPlan", LogicalPlanSerializer.convertToMap(appInfo.dag));
+ map.put("logicalPlan", LogicalPlanSerializer.convertToMap(appInfo.dag, false));
}
if (appInfo.error != null) {
map.put("error", appInfo.error);
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
index 90dd2b5..7b61d5b 100644
--- a/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
+++ b/engine/src/main/java/com/datatorrent/stram/codec/LogicalPlanSerializer.java
@@ -88,7 +88,7 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
* @param dag
* @return
*/
- public static Map<String, Object> convertToMap(LogicalPlan dag)
+ public static Map<String, Object> convertToMap(LogicalPlan dag, boolean includeModules)
{
HashMap<String, Object> result = new HashMap<String, Object>();
ArrayList<Object> operatorArray = new ArrayList< Object>();
@@ -200,6 +200,15 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
streamDetailMap.put("locality", streamMeta.getLocality().name());
}
}
+
+ if (includeModules) {
+ ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
+ result.put("modules", modulesArray);
+ for(LogicalPlan.ModuleMeta meta : dag.getAllModules()) {
+ modulesArray.add(getLogicalModuleDetails(dag, meta));
+ }
+ }
+
return result;
}
@@ -323,13 +332,43 @@ public class LogicalPlanSerializer extends JsonSerializer<LogicalPlan>
public static JSONObject convertToJsonObject(LogicalPlan dag)
{
- return new JSONObject(convertToMap(dag));
+ return new JSONObject(convertToMap(dag, false));
}
@Override
- public void serialize(LogicalPlan dag, JsonGenerator jg, SerializerProvider sp) throws IOException, JsonProcessingException
+ public void serialize(LogicalPlan dag, JsonGenerator jg, SerializerProvider sp) throws IOException,
+ JsonProcessingException
{
- jg.writeObject(convertToMap(dag));
+ jg.writeObject(convertToMap(dag, false));
+ }
+
+ /**
+ * Return information about operators and inner modules of a module.
+ *
+ * @param dag top level DAG
+ * @param moduleMeta module information. DAG within module is used for constructing response.
+ * @return
+ */
+ private static Map<String, Object> getLogicalModuleDetails(LogicalPlan dag, LogicalPlan.ModuleMeta moduleMeta)
+ {
+ Map<String, Object> moduleDetailMap = new HashMap<String, Object>();
+ ArrayList<String> operatorArray = new ArrayList<>();
+ moduleDetailMap.put("name", moduleMeta.getName());
+ moduleDetailMap.put("className", moduleMeta.getModule().getClass().getName());
+
+ moduleDetailMap.put("operators", operatorArray);
+ for (OperatorMeta operatorMeta : moduleMeta.getDag().getAllOperators()) {
+ if (operatorMeta.getModuleName() == null) {
+ String fullName = moduleMeta.getFullName() + LogicalPlan.MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName();
+ operatorArray.add(fullName);
+ }
+ }
+ ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
+ moduleDetailMap.put("modules", modulesArray);
+ for (LogicalPlan.ModuleMeta meta : moduleMeta.getDag().getAllModules()) {
+ modulesArray.add(getLogicalModuleDetails(dag, meta));
+ }
+ return moduleDetailMap;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 21039cc..377fa6d 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -32,6 +32,7 @@ import javax.validation.constraints.NotNull;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.slf4j.Logger;
@@ -1090,9 +1091,9 @@ public class LogicalPlan implements Serializable, DAG
}
// Avoid name conflict with module.
- if (modules.containsKey(name))
+ if (modules.containsKey(name)) {
throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
-
+ }
OperatorMeta decl = new OperatorMeta(name, operator);
rootOperators.add(decl); // will be removed when a sink is added to an input port for this operator
operators.put(name, decl);
@@ -1193,7 +1194,7 @@ public class LogicalPlan implements Serializable, DAG
* @param parentDAG parent dag to populate with operators from this and inner modules.
* @param conf configuration object.
*/
- public void flattenModule(LogicalPlan parentDAG, org.apache.hadoop.conf.Configuration conf)
+ public void flattenModule(LogicalPlan parentDAG, Configuration conf)
{
module.populateDAG(dag, conf);
for (ModuleMeta subModuleMeta : dag.getAllModules()) {
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 483576a..dbd3bc3 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -18,6 +18,7 @@
*/
package com.datatorrent.stram.plan.logical;
+
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -31,6 +32,7 @@ import java.lang.reflect.Type;
import java.util.*;
import java.util.Map.Entry;
+
import javax.validation.ValidationException;
import com.google.common.annotations.VisibleForTesting;
@@ -45,7 +47,6 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.collections.CollectionUtils;
@@ -60,6 +61,7 @@ import com.datatorrent.api.Context.DAGContext;
import com.datatorrent.api.Context.OperatorContext;
import com.datatorrent.api.Context.PortContext;
import com.datatorrent.api.annotation.ApplicationAnnotation;
+
import com.datatorrent.stram.StramUtils;
import com.datatorrent.stram.client.StramClientUtils;
import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/47b3ce8e/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
index 6fdba00..fd47d35 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/StramWebServices.java
@@ -68,6 +68,7 @@ import com.datatorrent.stram.StreamingContainerManager;
import com.datatorrent.stram.StringCodecs;
import com.datatorrent.stram.codec.LogicalPlanSerializer;
import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlan.ModuleMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
import com.datatorrent.stram.plan.logical.requests.LogicalPlanRequest;
@@ -561,17 +562,14 @@ public class StramWebServices
LOG.debug("Setting property for {}: {}={}", operatorName, key, val);
dagManager.setOperatorProperty(operatorName, key, val);
}
- }
- catch (JSONException ex) {
+ } catch (JSONException ex) {
LOG.warn("Got JSON Exception: ", ex);
- }
- catch (Exception ex) {
+ } catch (Exception ex) {
LOG.error("Caught exception: ", ex);
throw new RuntimeException(ex);
}
return response;
}
-
@POST // not supported by WebAppProxyServlet, can only be called directly
@Path(PATH_PHYSICAL_PLAN_OPERATORS + "/{operatorId:\\d+}/properties")
@Consumes(MediaType.APPLICATION_JSON)
@@ -633,60 +631,97 @@ public class StramWebServices
public JSONObject getPorts(@PathParam("operatorName") String operatorName)
{
OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
+ Set<LogicalPlan.InputPortMeta> inputPorts;
+ Set<LogicalPlan.OutputPortMeta> outputPorts;
if (logicalOperator == null) {
- throw new NotFoundException();
+ ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
+ if (logicalModule == null) {
+ throw new NotFoundException();
+ }
+ inputPorts = logicalModule.getInputStreams().keySet();
+ outputPorts = logicalModule.getOutputStreams().keySet();
+ } else {
+ inputPorts = logicalOperator.getInputStreams().keySet();
+ outputPorts = logicalOperator.getOutputStreams().keySet();
}
+
+ JSONObject result = getPortsObjects(inputPorts, outputPorts);
+ return result;
+ }
+
+ private JSONObject getPortsObjects(Collection<LogicalPlan.InputPortMeta> inputs, Collection<LogicalPlan.OutputPortMeta> outputs)
+ {
JSONObject result = new JSONObject();
JSONArray ports = new JSONArray();
try {
- for (LogicalPlan.InputPortMeta inputPort : logicalOperator.getInputStreams().keySet()) {
+ for (LogicalPlan.InputPortMeta inputPort : inputs) {
JSONObject port = new JSONObject();
port.put("name", inputPort.getPortName());
port.put("type", "input");
ports.put(port);
}
- for (LogicalPlan.OutputPortMeta outputPort : logicalOperator.getOutputStreams().keySet()) {
+ for (LogicalPlan.OutputPortMeta outputPort : outputs) {
JSONObject port = new JSONObject();
port.put("name", outputPort.getPortName());
port.put("type", "output");
ports.put(port);
}
result.put("ports", ports);
- }
- catch (JSONException ex) {
+ } catch (JSONException ex) {
throw new RuntimeException(ex);
}
return result;
}
+ private JSONObject getPortObject(Collection<LogicalPlan.InputPortMeta> inputs, Collection<LogicalPlan.OutputPortMeta> outputs,
+ String portName) throws JSONException
+ {
+ for (LogicalPlan.InputPortMeta inputPort : inputs) {
+ if (inputPort.getPortName().equals(portName)) {
+ JSONObject port = new JSONObject();
+ port.put("name", inputPort.getPortName());
+ port.put("type", "input");
+ return port;
+ }
+ }
+ for (LogicalPlan.OutputPortMeta outputPort : outputs) {
+ if (outputPort.getPortName().equals(portName)) {
+ JSONObject port = new JSONObject();
+ port.put("name", outputPort.getPortName());
+ port.put("type", "output");
+ return port;
+ }
+ }
+ return null;
+ }
+
+
@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports/{portName}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getPort(@PathParam("operatorName") String operatorName, @PathParam("portName") String portName)
{
OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
+ Set<LogicalPlan.InputPortMeta> inputPorts;
+ Set<LogicalPlan.OutputPortMeta> outputPorts;
if (logicalOperator == null) {
- throw new NotFoundException();
+ ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
+ if (logicalModule == null) {
+ throw new NotFoundException();
+ }
+ inputPorts = logicalModule.getInputStreams().keySet();
+ outputPorts = logicalModule.getOutputStreams().keySet();
+ } else {
+ inputPorts = logicalOperator.getInputStreams().keySet();
+ outputPorts = logicalOperator.getOutputStreams().keySet();
}
+
try {
- for (LogicalPlan.InputPortMeta inputPort : logicalOperator.getInputStreams().keySet()) {
- if (portName.equals(portName)) {
- JSONObject port = new JSONObject();
- port.put("name", inputPort.getPortName());
- port.put("type", "input");
- return port;
- }
- }
- for (LogicalPlan.OutputPortMeta outputPort : logicalOperator.getOutputStreams().keySet()) {
- if (portName.equals(portName)) {
- JSONObject port = new JSONObject();
- port.put("name", outputPort.getPortName());
- port.put("type", "output");
- return port;
- }
+ JSONObject resp = getPortObject(inputPorts, outputPorts, portName);
+ if (resp != null) {
+ return resp;
}
- }
- catch (JSONException ex) {
+ } catch (JSONException ex) {
throw new RuntimeException(ex);
}
throw new NotFoundException();
@@ -711,31 +746,41 @@ public class StramWebServices
{
init();
OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
+ BeanMap operatorProperties = null;
if (logicalOperator == null) {
- throw new NotFoundException();
+ ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
+ if (logicalModule == null) {
+ throw new NotFoundException();
+ }
+ operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalModule.getModule());
+ } else {
+ operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalOperator.getOperator());
}
- BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalOperator.getOperator());
+ Map<String, Object> m = getPropertiesAsMap(propertyName, operatorProperties);
+ return new JSONObject(objectMapper.writeValueAsString(m));
+ }
+
+ private Map<String, Object> getPropertiesAsMap(@QueryParam("propertyName") String propertyName, BeanMap operatorProperties)
+ {
Map<String, Object> m = new HashMap<String, Object>();
@SuppressWarnings("rawtypes")
Iterator entryIterator = operatorProperties.entryIterator();
while (entryIterator.hasNext()) {
try {
@SuppressWarnings("unchecked")
- Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
+ Entry<String, Object> entry = (Entry<String, Object>)entryIterator.next();
if (propertyName == null) {
m.put(entry.getKey(), entry.getValue());
- }
- else if (propertyName.equals(entry.getKey())) {
+ } else if (propertyName.equals(entry.getKey())) {
m.put(entry.getKey(), entry.getValue());
break;
}
- }
- catch (Exception ex) {
+ } catch (Exception ex) {
LOG.warn("Caught exception", ex);
}
}
- return new JSONObject(objectMapper.writeValueAsString(m));
+ return m;
}
@GET
@@ -765,10 +810,10 @@ public class StramWebServices
@GET
@Path(PATH_LOGICAL_PLAN)
@Produces(MediaType.APPLICATION_JSON)
- public JSONObject getLogicalPlan() throws JSONException, IOException
+ public JSONObject getLogicalPlan(@QueryParam("includeModules") String includeModules) throws JSONException, IOException
{
- LogicalPlan lp = dagManager.getLogicalPlan();
- return new JSONObject(objectMapper.writeValueAsString(LogicalPlanSerializer.convertToMap(lp)));
+ return new JSONObject(objectMapper.writeValueAsString(LogicalPlanSerializer.convertToMap(
+ dagManager.getLogicalPlan(), includeModules != null)));
}
@POST // not supported by WebAppProxyServlet, can only be called directly
[3/5] incubator-apex-core git commit: APEXCORE-104 Added flattening
of module into parent DAG
Posted by th...@apache.org.
APEXCORE-104 Added flattening of module into parent DAG
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/14a09bb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/14a09bb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/14a09bb5
Branch: refs/heads/devel-3
Commit: 14a09bb51cb30585e7979ec022ec762ac2ba91e5
Parents: b0360d4
Author: chinmaykolhatkar <ch...@datatorrent.com>
Authored: Wed Oct 7 15:06:36 2015 +0530
Committer: chinmaykolhatkar <ch...@datatorrent.com>
Committed: Tue Dec 22 01:42:43 2015 +0530
----------------------------------------------------------------------
api/src/main/java/com/datatorrent/api/DAG.java | 4 -
.../stram/plan/logical/LogicalPlan.java | 169 +++++++++++++++----
.../plan/logical/LogicalPlanConfiguration.java | 8 +
3 files changed, 146 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/14a09bb5/api/src/main/java/com/datatorrent/api/DAG.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/DAG.java b/api/src/main/java/com/datatorrent/api/DAG.java
index 1dce402..7c793f9 100644
--- a/api/src/main/java/com/datatorrent/api/DAG.java
+++ b/api/src/main/java/com/datatorrent/api/DAG.java
@@ -165,10 +165,6 @@ public interface DAG extends DAGContext, Serializable
String getName();
Module getModule();
-
- InputPortMeta getMeta(Operator.InputPort<?> port);
-
- OutputPortMeta getMeta(Operator.OutputPort<?> port);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/14a09bb5/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 53e81bc..5a3e167 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -80,6 +80,7 @@ public class LogicalPlan implements Serializable, DAG
public static final String SER_FILE_NAME = "dt-conf.ser";
public static final String LAUNCH_CONFIG_FILE_NAME = "dt-launch-config.xml";
private static final transient AtomicInteger logicalOperatorSequencer = new AtomicInteger();
+ public static final String MODULE_NAMESPACE_SEPARATOR = "$";
/**
* Constant
@@ -146,6 +147,7 @@ public class LogicalPlan implements Serializable, DAG
private final Map<String, StreamMeta> streams = new HashMap<String, StreamMeta>();
private final Map<String, OperatorMeta> operators = new HashMap<String, OperatorMeta>();
+ public final Map<String, ModuleMeta> modules = new LinkedHashMap<>();
private final List<OperatorMeta> rootOperators = new ArrayList<OperatorMeta>();
private final Attribute.AttributeMap attributes = new DefaultAttributeMap();
private transient int nodeIndex = 0; // used for cycle validation
@@ -733,6 +735,7 @@ public class LogicalPlan implements Serializable, DAG
private transient Integer lowlink; // for cycle detection
private transient Operator operator;
private MetricAggregatorMeta metricAggregatorMeta;
+ private String moduleName; // Name of the module which has this operator. null if this is a top level operator.
/*
* Used for OIO validation,
@@ -819,6 +822,16 @@ public class LogicalPlan implements Serializable, DAG
return metricAggregatorMeta;
}
+ public String getModuleName()
+ {
+ return moduleName;
+ }
+
+ public void setModuleName(String moduleName)
+ {
+ this.moduleName = moduleName;
+ }
+
protected void populateAggregatorMeta()
{
AutoMetric.Aggregator aggregator = getValue(OperatorContext.METRICS_AGGREGATOR);
@@ -1073,37 +1086,38 @@ public class LogicalPlan implements Serializable, DAG
throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
}
+ // Avoid name conflict with module.
+ if (modules.containsKey(name))
+ throw new IllegalArgumentException("duplicate operator id: " + operators.get(name));
+
OperatorMeta decl = new OperatorMeta(name, operator);
rootOperators.add(decl); // will be removed when a sink is added to an input port for this operator
operators.put(name, decl);
return operator;
}
+ /**
+ * Module meta object.
+ */
public final class ModuleMeta implements DAG.ModuleMeta, Serializable
{
private final LinkedHashMap<InputPortMeta, StreamMeta> inputStreams = new LinkedHashMap<>();
private final LinkedHashMap<OutputPortMeta, StreamMeta> outputStreams = new LinkedHashMap<>();
private final Attribute.AttributeMap attributes;
- @SuppressWarnings("unused")
- private final int id;
@NotNull
- private final String name;
- private transient Integer nindex; // for cycle detection
- private transient Integer lowlink; // for cycle detection
+ private String name;
private transient Module module;
+ private ModuleMeta parent;
+ private LogicalPlan dag = null;
+ private transient String fullName;
- public ModuleMeta(String name, Module module)
- {
- this(name, module, new DefaultAttributeMap());
- }
-
- public ModuleMeta(String name, Module module, DefaultAttributeMap attributeMap)
+ private ModuleMeta(String name, Module module)
{
LOG.debug("Initializing {} as {}", name, module.getClass().getName());
this.name = name;
this.module = module;
- this.id = logicalOperatorSequencer.decrementAndGet();
- this.attributes = attributeMap;
+ this.attributes = new DefaultAttributeMap();
+ this.dag = new LogicalPlan();
}
@Override
@@ -1119,43 +1133,104 @@ public class LogicalPlan implements Serializable, DAG
}
@Override
- public DAG.InputPortMeta getMeta(InputPort<?> port)
+ public Attribute.AttributeMap getAttributes()
{
- return null;
+ return attributes;
}
@Override
- public DAG.OutputPortMeta getMeta(OutputPort<?> port)
+ public <T> T getValue(Attribute<T> key)
{
- return null;
+ return attributes.get(key);
}
@Override
- public Attribute.AttributeMap getAttributes()
+ public void setCounters(Object counters)
{
- return null;
+
}
@Override
- public <T> T getValue(Attribute<T> key)
+ public void sendMetrics(Collection<String> metricNames)
{
- return null;
+
}
- @Override
- public void setCounters(Object counters)
+ public LinkedHashMap<InputPortMeta, StreamMeta> getInputStreams()
{
+ return inputStreams;
+ }
+ public LinkedHashMap<OutputPortMeta, StreamMeta> getOutputStreams()
+ {
+ return outputStreams;
}
- @Override
- public void sendMetrics(Collection<String> metricNames)
+ public LogicalPlan getDag()
{
+ return dag;
+ }
+ private void writeObject(ObjectOutputStream out) throws IOException
+ {
+ out.defaultWriteObject();
+ FSStorageAgent.store(out, module);
}
- }
- public transient Map<String, ModuleMeta> modules = Maps.newHashMap();
+ private void readObject(ObjectInputStream input) throws IOException, ClassNotFoundException
+ {
+ input.defaultReadObject();
+ module = (Module)FSStorageAgent.retrieve(input);
+ }
+
+ /**
+ * Expand the module and add its operator to the parentDAG. After this method finishes the module is expanded fully
+ * with all its submodules also expanded. The parentDAG contains the operator added by all the modules.
+ *
+ * @param parentDAG parent dag to populate with operators from this and inner modules.
+ * @param conf configuration object.
+ */
+ public void flattenModule(LogicalPlan parentDAG, org.apache.hadoop.conf.Configuration conf)
+ {
+ module.populateDAG(dag, conf);
+ for (ModuleMeta subModuleMeta : dag.getAllModules()) {
+ subModuleMeta.setParent(this);
+ subModuleMeta.flattenModule(dag, conf);
+ }
+ parentDAG.addDAGToCurrentDAG(this);
+ }
+
+ /**
+ * Return full name of the module. If this is a inner module, i.e module inside of module this method will traverse
+ * till the top level module, and construct the name by concatenating name of modules in the chain in reverse order
+ * separated by MODULE_NAMESPACE_SEPARATO.
+ *
+ * For example If there is module M1, which adds another module M2 in the DAG. Then the full name of the module M2
+ * is ("M1" ++ MODULE_NAMESPACE_SEPARATO + "M2")
+ *
+ * @return full name of the module.
+ */
+ public String getFullName()
+ {
+ if (fullName != null) {
+ return fullName;
+ }
+
+ if (parent == null) {
+ fullName = name;
+ } else {
+ fullName = parent.getFullName() + MODULE_NAMESPACE_SEPARATOR + name;
+ }
+ return fullName;
+ }
+
+ private void setParent(ModuleMeta meta)
+ {
+ this.parent = meta;
+ }
+
+ private static final long serialVersionUID = 7562277769188329223L;
+ }
@Override
public <T extends Module> T addModule(String name, T module)
@@ -1166,6 +1241,10 @@ public class LogicalPlan implements Serializable, DAG
}
throw new IllegalArgumentException("duplicate module is: " + modules.get(name));
}
+ if (operators.containsKey(name)) {
+ throw new IllegalArgumentException("duplicate module is: " + modules.get(name));
+ }
+
ModuleMeta meta = new ModuleMeta(name, module);
modules.put(name, meta);
return module;
@@ -1228,6 +1307,33 @@ public class LogicalPlan implements Serializable, DAG
return s;
}
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private void addDAGToCurrentDAG(ModuleMeta moduleMeta)
+ {
+ LogicalPlan subDag = moduleMeta.getDag();
+ String subDAGName = moduleMeta.getName();
+ String name;
+ for (OperatorMeta operatorMeta : subDag.getAllOperators()) {
+ name = subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName();
+ this.addOperator(name, operatorMeta.getOperator());
+ OperatorMeta operatorMetaNew = this.getOperatorMeta(name);
+ operatorMetaNew.setModuleName(operatorMeta.getModuleName() == null ? subDAGName :
+ subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getModuleName());
+ }
+
+ for (StreamMeta streamMeta : subDag.getAllStreams()) {
+ OutputPortMeta sourceMeta = streamMeta.getSource();
+ List<InputPort<?>> ports = new LinkedList<>();
+ for (InputPortMeta inputPortMeta : streamMeta.getSinks()) {
+ ports.add(inputPortMeta.getPortObject());
+ }
+ InputPort[] inputPorts = ports.toArray(new InputPort[]{});
+
+ name = subDAGName + MODULE_NAMESPACE_SEPARATOR + streamMeta.getName();
+ this.addStream(name, sourceMeta.getPortObject(), inputPorts);
+ }
+ }
+
@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1)
@@ -1276,7 +1382,7 @@ public class LogicalPlan implements Serializable, DAG
private OutputPortMeta assertGetPortMeta(Operator.OutputPort<?> port)
{
- for (OperatorMeta o: getAllOperators()) {
+ for (OperatorMeta o : getAllOperators()) {
OutputPortMeta opm = o.getPortMapping().outPortMap.get(port);
if (opm != null) {
return opm;
@@ -1287,7 +1393,7 @@ public class LogicalPlan implements Serializable, DAG
private InputPortMeta assertGetPortMeta(Operator.InputPort<?> port)
{
- for (OperatorMeta o: getAllOperators()) {
+ for (OperatorMeta o : getAllOperators()) {
InputPortMeta opm = o.getPortMapping().inPortMap.get(port);
if (opm != null) {
return opm;
@@ -1324,7 +1430,8 @@ public class LogicalPlan implements Serializable, DAG
return Collections.unmodifiableCollection(this.operators.values());
}
- public Collection<ModuleMeta> getAllModules() {
+ public Collection<ModuleMeta> getAllModules()
+ {
return Collections.unmodifiableCollection(this.modules.values());
}
@@ -1341,7 +1448,7 @@ public class LogicalPlan implements Serializable, DAG
public ModuleMeta getModuleMeta(String moduleName)
{
- return null;
+ return this.modules.get(moduleName);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/14a09bb5/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 9bbe85c..6dc4c0c 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -2116,12 +2116,20 @@ public class LogicalPlanConfiguration {
// Expand the modules within the dag recursively
setModuleProperties(dag, appName);
+ flattenDAG(dag, conf);
// inject external operator configuration
setOperatorConfiguration(dag, appConfs, appName);
setStreamConfiguration(dag, appConfs, appName);
}
+ private void flattenDAG(LogicalPlan dag, Configuration conf)
+ {
+ for (ModuleMeta moduleMeta : dag.getAllModules()) {
+ moduleMeta.flattenModule(dag, conf);
+ }
+ }
+
public static Properties readProperties(String filePath) throws IOException
{
InputStream is = new FileInputStream(filePath);
[4/5] incubator-apex-core git commit: APEXCORE-194 Added support for
proxy ports Added test cases.
Posted by th...@apache.org.
APEXCORE-194 Added support for proxy ports
Added test cases.
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/c1314eaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/c1314eaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/c1314eaf
Branch: refs/heads/devel-3
Commit: c1314eafaac239b420d085a4584d5c5acaf3e69b
Parents: 14a09bb
Author: bhupeshchawda <bh...@gmail.com>
Authored: Tue Oct 6 12:34:24 2015 +0530
Committer: Tushar R. Gosavi <tu...@apache.org>
Committed: Tue Dec 22 02:04:18 2015 +0530
----------------------------------------------------------------------
.../main/java/com/datatorrent/api/Module.java | 120 ++++
.../stram/plan/logical/LogicalPlan.java | 51 +-
.../plan/logical/LogicalPlanConfiguration.java | 1 +
.../plan/logical/module/ModuleAppTest.java | 168 ++++++
.../logical/module/TestModuleExpansion.java | 552 +++++++++++++++++++
5 files changed, 888 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/api/src/main/java/com/datatorrent/api/Module.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Module.java b/api/src/main/java/com/datatorrent/api/Module.java
index 1220fc1..67682e7 100644
--- a/api/src/main/java/com/datatorrent/api/Module.java
+++ b/api/src/main/java/com/datatorrent/api/Module.java
@@ -21,8 +21,128 @@ package com.datatorrent.api;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.Operator.InputPort;
+import com.datatorrent.api.Operator.OutputPort;
+import com.datatorrent.api.Operator.Unifier;
+
+/**
+ * A Module is a component which can be added to the DAG similar to the operator,
+ * using addModule API. The module should implement populateDAG method, which
+ * will be called by the platform, and DAG populated by the module will be
+ * replace in place of the module.
+ *
+ */
@InterfaceStability.Evolving
public interface Module
{
void populateDAG(DAG dag, Configuration conf);
+
+ /**
+ * These ports allow platform to short circuit module port to the operator port. i.e When a module is expanded, it can
+ * specify which operator's port is used to replaced the module port in the final DAG.
+ *
+ * @param <T> data type accepted at the input port.
+ */
+ final class ProxyInputPort<T> implements InputPort<T>
+ {
+ InputPort<T> inputPort;
+
+ public void set(InputPort<T> port)
+ {
+ inputPort = port;
+ }
+
+ public InputPort<T> get()
+ {
+ return inputPort;
+ }
+
+ @Override
+ public void setup(PortContext context)
+ {
+ if (inputPort != null) {
+ inputPort.setup(context);
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ if (inputPort != null) {
+ inputPort.teardown();
+ }
+ }
+
+ @Override
+ public Sink<T> getSink()
+ {
+ return inputPort == null ? null : inputPort.getSink();
+ }
+
+ @Override
+ public void setConnected(boolean connected)
+ {
+ if (inputPort != null) {
+ inputPort.setConnected(connected);
+ }
+ }
+
+ @Override
+ public StreamCodec<T> getStreamCodec()
+ {
+ return inputPort == null ? null : inputPort.getStreamCodec();
+ }
+ }
+
+ /**
+ * Similar to ProxyInputPort, but on output side.
+ *
+ * @param <T> datatype emitted on the port.
+ */
+ final class ProxyOutputPort<T> implements OutputPort<T>
+ {
+ OutputPort<T> outputPort;
+
+ public void set(OutputPort<T> port)
+ {
+ outputPort = port;
+ }
+
+ public OutputPort<T> get()
+ {
+ return outputPort;
+ }
+
+ @Override
+ public void setup(PortContext context)
+ {
+ if (outputPort != null) {
+ outputPort.setup(context);
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ if (outputPort != null) {
+ outputPort.teardown();
+ }
+ }
+
+ @Override
+ public void setSink(Sink<Object> s)
+ {
+ if (outputPort != null) {
+ outputPort.setSink(s);
+ }
+ }
+
+ @Override
+ public Unifier<T> getUnifier()
+ {
+ return outputPort == null ? null : outputPort.getUnifier();
+ }
+ }
}
+
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 5a3e167..21039cc 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -37,12 +37,14 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Sets;
import com.datatorrent.api.*;
import com.datatorrent.api.Attribute.AttributeMap;
import com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap;
+import com.datatorrent.api.Module.ProxyInputPort;
+import com.datatorrent.api.Module.ProxyOutputPort;
import com.datatorrent.api.Operator.InputPort;
import com.datatorrent.api.Operator.OutputPort;
import com.datatorrent.api.Operator.Unifier;
@@ -152,6 +154,7 @@ public class LogicalPlan implements Serializable, DAG
private final Attribute.AttributeMap attributes = new DefaultAttributeMap();
private transient int nodeIndex = 0; // used for cycle validation
private transient Stack<OperatorMeta> stack = new Stack<OperatorMeta>(); // used for cycle validation
+ private transient Map<String, ArrayListMultimap<OutputPort<?>, InputPort<?>>> streamLinks = new HashMap<>();
@Override
public Attribute.AttributeMap getAttributes()
@@ -1197,6 +1200,7 @@ public class LogicalPlan implements Serializable, DAG
subModuleMeta.setParent(this);
subModuleMeta.flattenModule(dag, conf);
}
+ dag.applyStreamLinks();
parentDAG.addDAGToCurrentDAG(this);
}
@@ -1300,13 +1304,52 @@ public class LogicalPlan implements Serializable, DAG
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T>... sinks)
{
StreamMeta s = addStream(id);
- s.setSource(source);
- for (Operator.InputPort<?> sink: sinks) {
- s.addSink(sink);
+ id = s.id;
+ ArrayListMultimap<OutputPort<?>, InputPort<?>> streamMap = ArrayListMultimap.create();
+ if (!(source instanceof ProxyOutputPort)) {
+ s.setSource(source);
+ }
+ for (Operator.InputPort<?> sink : sinks) {
+ if (source instanceof ProxyOutputPort || sink instanceof ProxyInputPort) {
+ streamMap.put(source, sink);
+ streamLinks.put(id, streamMap);
+ } else {
+ if (s.getSource() == null) {
+ s.setSource(source);
+ }
+ s.addSink(sink);
+ }
}
return s;
}
+ /**
+ * This will be called once the Logical Dag is expanded, and the proxy input and proxy output ports are populated with
+ * the actual ports that they refer to This method adds sources and sinks for the StreamMeta objects which were left
+ * empty in the addStream call.
+ */
+ public void applyStreamLinks()
+ {
+ for (String id : streamLinks.keySet()) {
+ StreamMeta s = getStream(id);
+ for (Map.Entry<Operator.OutputPort<?>, Operator.InputPort<?>> pair : streamLinks.get(id).entries()) {
+ if (s.getSource() == null) {
+ Operator.OutputPort<?> outputPort = pair.getKey();
+ while (outputPort instanceof ProxyOutputPort) {
+ outputPort = ((ProxyOutputPort<?>)outputPort).get();
+ }
+ s.setSource(outputPort);
+ }
+
+ Operator.InputPort<?> inputPort = pair.getValue();
+ while (inputPort instanceof ProxyInputPort) {
+ inputPort = ((ProxyInputPort<?>)inputPort).get();
+ }
+ s.addSink(inputPort);
+ }
+ }
+ }
+
@SuppressWarnings({ "unchecked", "rawtypes" })
private void addDAGToCurrentDAG(ModuleMeta moduleMeta)
{
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 6dc4c0c..483576a 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -2128,6 +2128,7 @@ public class LogicalPlanConfiguration {
for (ModuleMeta moduleMeta : dag.getAllModules()) {
moduleMeta.flattenModule(dag, conf);
}
+ dag.applyStreamLinks();
}
public static Properties readProperties(String filePath) throws IOException
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java
new file mode 100644
index 0000000..97c015e
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/ModuleAppTest.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.Module;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+
+/**
+ * Unit tests for testing Dag expansion with modules and proxy port substitution
+ */
+public class ModuleAppTest
+{
+
+ /*
+ * Input Operator - 1
+ */
+ static class DummyInputOperator extends BaseOperator implements InputOperator
+ {
+
+ Random r = new Random();
+ public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void emitTuples()
+ {
+ output.emit(r.nextInt());
+ }
+ }
+
+ /*
+ * Input Operator - 1.1
+ */
+ static class DummyOperatorAfterInput extends BaseOperator
+ {
+
+ public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
+ {
+ @Override
+ public void process(Integer tuple)
+ {
+ output.emit(tuple);
+ }
+ };
+ public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<Integer>();
+ }
+
+ /*
+ * Operator - 2
+ */
+ static class DummyOperator extends BaseOperator
+ {
+ int prop;
+
+ public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
+ {
+ @Override
+ public void process(Integer tuple)
+ {
+ LOG.debug(tuple.intValue() + " processed");
+ output.emit(tuple);
+ }
+ };
+ public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<Integer>();
+ }
+
+ /*
+ * Output Operator - 3
+ */
+ static class DummyOutputOperator extends BaseOperator
+ {
+ int prop;
+
+ public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
+ {
+ @Override
+ public void process(Integer tuple)
+ {
+ LOG.debug(tuple.intValue() + " processed");
+ }
+ };
+ }
+
+ /*
+ * Module Definition
+ */
+ static class TestModule implements Module
+ {
+
+ public transient ProxyInputPort<Integer> moduleInput = new Module.ProxyInputPort<Integer>();
+ public transient ProxyOutputPort<Integer> moduleOutput = new Module.ProxyOutputPort<Integer>();
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ LOG.debug("Module - PopulateDAG");
+ DummyOperator dummyOperator = dag.addOperator("DummyOperator", new DummyOperator());
+ moduleInput.set(dummyOperator.input);
+ moduleOutput.set(dummyOperator.output);
+ }
+ }
+
+ static class Application implements StreamingApplication
+ {
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ LOG.debug("Application - PopulateDAG");
+ DummyInputOperator dummyInputOperator = dag.addOperator("DummyInputOperator", new DummyInputOperator());
+ DummyOperatorAfterInput dummyOperatorAfterInput = dag.addOperator("DummyOperatorAfterInput",
+ new DummyOperatorAfterInput());
+ Module m1 = dag.addModule("TestModule1", new TestModule());
+ Module m2 = dag.addModule("TestModule2", new TestModule());
+ DummyOutputOperator dummyOutputOperator = dag.addOperator("DummyOutputOperator", new DummyOutputOperator());
+ dag.addStream("Operator To Operator", dummyInputOperator.output, dummyOperatorAfterInput.input);
+ dag.addStream("Operator To Module", dummyOperatorAfterInput.output, ((TestModule)m1).moduleInput);
+ dag.addStream("Module To Module", ((TestModule)m1).moduleOutput, ((TestModule)m2).moduleInput);
+ dag.addStream("Module To Operator", ((TestModule)m2).moduleOutput, dummyOutputOperator.input);
+ }
+ }
+
+ @Test
+ public void validateTestApplication()
+ {
+ Configuration conf = new Configuration(false);
+ LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = new LogicalPlan();
+ lpc.prepareDAG(dag, new Application(), "TestApp");
+
+ Assert.assertEquals(2, dag.getAllModules().size(), 2);
+ Assert.assertEquals(5, dag.getAllOperators().size());
+ Assert.assertEquals(4, dag.getAllStreams().size());
+ dag.validate();
+ }
+
+ private static Logger LOG = LoggerFactory.getLogger(ModuleAppTest.class);
+}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/c1314eaf/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java
new file mode 100644
index 0000000..5bfd8f1
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/module/TestModuleExpansion.java
@@ -0,0 +1,552 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.plan.logical.module;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.Module;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+
+public class TestModuleExpansion
+{
+ static class DummyInputOperator extends BaseOperator implements InputOperator
+ {
+ private int inputOperatorProp = 0;
+
+ Random r = new Random();
+ public transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void emitTuples()
+ {
+ out.emit(r.nextInt());
+ }
+
+ public int getInputOperatorProp()
+ {
+ return inputOperatorProp;
+ }
+
+ public void setInputOperatorProp(int inputOperatorProp)
+ {
+ this.inputOperatorProp = inputOperatorProp;
+ }
+ }
+
+ static class DummyOperator extends BaseOperator
+ {
+ private int operatorProp = 0;
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Integer> out1 = new DefaultOutputPort<>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Integer> out2 = new DefaultOutputPort<>();
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient DefaultInputPort<Integer> in = new DefaultInputPort<Integer>()
+ {
+ @Override
+ public void process(Integer tuple)
+ {
+ out1.emit(tuple);
+ out2.emit(tuple);
+ }
+ };
+
+ public int getOperatorProp()
+ {
+ return operatorProp;
+ }
+
+ public void setOperatorProp(int operatorProp)
+ {
+ this.operatorProp = operatorProp;
+ }
+ }
+
+ static class Level1Module implements Module
+ {
+ private int level1ModuleProp = 0;
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient ProxyOutputPort<Integer> mOut = new ProxyOutputPort<>();
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ DummyOperator o1 = dag.addOperator("O1", new DummyOperator());
+ o1.setOperatorProp(level1ModuleProp);
+ mIn.set(o1.in);
+ mOut.set(o1.out1);
+ }
+
+ public int getLevel1ModuleProp()
+ {
+ return level1ModuleProp;
+ }
+
+ public void setLevel1ModuleProp(int level1ModuleProp)
+ {
+ this.level1ModuleProp = level1ModuleProp;
+ }
+ }
+
+ static class Level2ModuleA implements Module
+ {
+ private int level2ModuleAProp1 = 0;
+ private int level2ModuleAProp2 = 0;
+ private int level2ModuleAProp3 = 0;
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient ProxyOutputPort<Integer> mOut1 = new ProxyOutputPort<>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient ProxyOutputPort<Integer> mOut2 = new ProxyOutputPort<>();
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ Level1Module m1 = dag.addModule("M1", new Level1Module());
+ m1.setLevel1ModuleProp(level2ModuleAProp1);
+
+ Level1Module m2 = dag.addModule("M2", new Level1Module());
+ m2.setLevel1ModuleProp(level2ModuleAProp2);
+
+ DummyOperator o1 = dag.addOperator("O1", new DummyOperator());
+ o1.setOperatorProp(level2ModuleAProp3);
+
+ dag.addStream("M1_M2&O1", m1.mOut, m2.mIn, o1.in);
+
+ mIn.set(m1.mIn);
+ mOut1.set(m2.mOut);
+ mOut2.set(o1.out1);
+ }
+
+ public int getLevel2ModuleAProp1()
+ {
+ return level2ModuleAProp1;
+ }
+
+ public void setLevel2ModuleAProp1(int level2ModuleAProp1)
+ {
+ this.level2ModuleAProp1 = level2ModuleAProp1;
+ }
+
+ public int getLevel2ModuleAProp2()
+ {
+ return level2ModuleAProp2;
+ }
+
+ public void setLevel2ModuleAProp2(int level2ModuleAProp2)
+ {
+ this.level2ModuleAProp2 = level2ModuleAProp2;
+ }
+
+ public int getLevel2ModuleAProp3()
+ {
+ return level2ModuleAProp3;
+ }
+
+ public void setLevel2ModuleAProp3(int level2ModuleAProp3)
+ {
+ this.level2ModuleAProp3 = level2ModuleAProp3;
+ }
+ }
+
+ static class Level2ModuleB implements Module
+ {
+ private int level2ModuleBProp1 = 0;
+ private int level2ModuleBProp2 = 0;
+ private int level2ModuleBProp3 = 0;
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient ProxyOutputPort<Integer> mOut1 = new ProxyOutputPort<>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient ProxyOutputPort<Integer> mOut2 = new ProxyOutputPort<>();
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ DummyOperator o1 = dag.addOperator("O1", new DummyOperator());
+ o1.setOperatorProp(level2ModuleBProp1);
+
+ Level1Module m1 = dag.addModule("M1", new Level1Module());
+ m1.setLevel1ModuleProp(level2ModuleBProp2);
+
+ DummyOperator o2 = dag.addOperator("O2", new DummyOperator());
+ o2.setOperatorProp(level2ModuleBProp3);
+
+ dag.addStream("O1_M1", o1.out1, m1.mIn);
+ dag.addStream("O1_O2", o1.out2, o2.in);
+
+ mIn.set(o1.in);
+ mOut1.set(m1.mOut);
+ mOut2.set(o2.out1);
+ }
+
+ public int getLevel2ModuleBProp1()
+ {
+ return level2ModuleBProp1;
+ }
+
+ public void setLevel2ModuleBProp1(int level2ModuleBProp1)
+ {
+ this.level2ModuleBProp1 = level2ModuleBProp1;
+ }
+
+ public int getLevel2ModuleBProp2()
+ {
+ return level2ModuleBProp2;
+ }
+
+ public void setLevel2ModuleBProp2(int level2ModuleBProp2)
+ {
+ this.level2ModuleBProp2 = level2ModuleBProp2;
+ }
+
+ public int getLevel2ModuleBProp3()
+ {
+ return level2ModuleBProp3;
+ }
+
+ public void setLevel2ModuleBProp3(int level2ModuleBProp3)
+ {
+ this.level2ModuleBProp3 = level2ModuleBProp3;
+ }
+ }
+
+ static class Level3Module implements Module
+ {
+
+ public final transient ProxyInputPort<Integer> mIn = new ProxyInputPort<>();
+ public final transient ProxyOutputPort<Integer> mOut1 = new ProxyOutputPort<>();
+ public final transient ProxyOutputPort<Integer> mOut2 = new ProxyOutputPort<>();
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ DummyOperator op = dag.addOperator("O1", new DummyOperator());
+ Level2ModuleB m1 = dag.addModule("M1", new Level2ModuleB());
+ Level1Module m2 = dag.addModule("M2", new Level1Module());
+
+ dag.addStream("s1", op.out1, m1.mIn);
+ dag.addStream("s2", op.out2, m2.mIn);
+
+ mIn.set(op.in);
+ mOut1.set(m1.mOut1);
+ mOut2.set(m2.mOut);
+ }
+ }
+
+ static class NestedModuleApp implements StreamingApplication
+ {
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ DummyInputOperator o1 = dag.addOperator("O1", new DummyInputOperator());
+ o1.setInputOperatorProp(1);
+
+ DummyOperator o2 = dag.addOperator("O2", new DummyOperator());
+ o2.setOperatorProp(2);
+
+ Level2ModuleA ma = dag.addModule("Ma", new Level2ModuleA());
+ ma.setLevel2ModuleAProp1(11);
+ ma.setLevel2ModuleAProp2(12);
+ ma.setLevel2ModuleAProp3(13);
+
+ Level2ModuleB mb = dag.addModule("Mb", new Level2ModuleB());
+ mb.setLevel2ModuleBProp1(21);
+ mb.setLevel2ModuleBProp2(22);
+ mb.setLevel2ModuleBProp3(23);
+
+ Level2ModuleA mc = dag.addModule("Mc", new Level2ModuleA());
+ mc.setLevel2ModuleAProp1(31);
+ mc.setLevel2ModuleAProp2(32);
+ mc.setLevel2ModuleAProp3(33);
+
+ Level2ModuleB md = dag.addModule("Md", new Level2ModuleB());
+ md.setLevel2ModuleBProp1(41);
+ md.setLevel2ModuleBProp2(42);
+ md.setLevel2ModuleBProp3(43);
+
+ Level3Module me = dag.addModule("Me", new Level3Module());
+
+ dag.addStream("O1_O2", o1.out, o2.in, me.mIn);
+ dag.addStream("O2_Ma", o2.out1, ma.mIn);
+ dag.addStream("Ma_Mb", ma.mOut1, mb.mIn);
+ dag.addStream("Ma_Md", ma.mOut2, md.mIn);
+ dag.addStream("Mb_Mc", mb.mOut2, mc.mIn);
+ }
+ }
+
+ @Test
+ public void testModuleExtreme()
+ {
+ StreamingApplication app = new NestedModuleApp();
+ Configuration conf = new Configuration(false);
+ LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = new LogicalPlan();
+ lpc.prepareDAG(dag, app, "ModuleApp");
+
+ dag.validate();
+ validateTopLevelOperators(dag);
+ validateTopLevelStreams(dag);
+ validatePublicMethods(dag);
+ }
+
+ private void validateTopLevelStreams(LogicalPlan dag)
+ {
+ List<String> streamNames = new ArrayList<>();
+ for (LogicalPlan.StreamMeta streamMeta : dag.getAllStreams()) {
+ streamNames.add(streamMeta.getName());
+ }
+
+ Assert.assertTrue(streamNames.contains(componentName("Mb", "O1_M1")));
+ Assert.assertTrue(streamNames.contains("O2_Ma"));
+ Assert.assertTrue(streamNames.contains("Mb_Mc"));
+ Assert.assertTrue(streamNames.contains(componentName("Mb", "O1_O2")));
+ Assert.assertTrue(streamNames.contains(componentName("Ma", "M1_M2&O1")));
+ Assert.assertTrue(streamNames.contains(componentName("Md", "O1_M1")));
+ Assert.assertTrue(streamNames.contains(componentName("Ma_Md")));
+ Assert.assertTrue(streamNames.contains(componentName("Mc", "M1_M2&O1")));
+ Assert.assertTrue(streamNames.contains(componentName("Md", "O1_O2")));
+ Assert.assertTrue(streamNames.contains("Ma_Mb"));
+ Assert.assertTrue(streamNames.contains("O1_O2"));
+
+ validateSeperateStream(dag, componentName("Mb", "O1_M1"), componentName("Mb", "O1"),
+ componentName("Mb", "M1", "O1"));
+ validateSeperateStream(dag, "O2_Ma", "O2", componentName("Ma", "M1", "O1"));
+ validateSeperateStream(dag, "Mb_Mc", componentName("Mb", "O2"), componentName("Mc", "M1", "O1"));
+ validateSeperateStream(dag, componentName("Mb", "O1_O2"), componentName("Mb", "O1"), componentName("Mb", "O2"));
+ validateSeperateStream(dag, componentName("Ma", "M1_M2&O1"), componentName("Ma", "M1", "O1"),
+ componentName("Ma", "O1"), componentName("Ma", "M2", "O1"));
+ validateSeperateStream(dag, componentName("Md", "O1_M1"), componentName("Md", "O1"),
+ componentName("Md", "M1", "O1"));
+ validateSeperateStream(dag, "Ma_Md", componentName("Ma", "O1"), componentName("Md", "O1"));
+ validateSeperateStream(dag, componentName("Mc", "M1_M2&O1"), componentName("Mc", "M1", "O1"),
+ componentName("Mc", "O1"), componentName("Mc", "M2", "O1"));
+ validateSeperateStream(dag, componentName("Md", "O1_O2"), componentName("Md", "O1"), componentName("Md", "O2"));
+ validateSeperateStream(dag, "Ma_Mb", componentName("Ma", "M2", "O1"), componentName("Mb", "O1"));
+ validateSeperateStream(dag, "O1_O2", "O1", "O2", componentName("Me", "O1"));
+ }
+
+ private void validateSeperateStream(LogicalPlan dag, String streamName, String inputOperatorName,
+ String... outputOperatorNames)
+ {
+ LogicalPlan.StreamMeta streamMeta = dag.getStream(streamName);
+ String sourceName = streamMeta.getSource().getOperatorMeta().getName();
+
+ List<String> sinksName = new ArrayList<>();
+ for (LogicalPlan.InputPortMeta inputPortMeta : streamMeta.getSinks()) {
+ sinksName.add(inputPortMeta.getOperatorWrapper().getName());
+ }
+
+ Assert.assertTrue(inputOperatorName.equals(sourceName));
+ Assert.assertEquals(outputOperatorNames.length, sinksName.size());
+
+ for (String outputOperatorName : outputOperatorNames) {
+ Assert.assertTrue(sinksName.contains(outputOperatorName));
+ }
+ }
+
+ private void validateTopLevelOperators(LogicalPlan dag)
+ {
+ List<String> operatorNames = new ArrayList<>();
+ for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
+ operatorNames.add(operatorMeta.getName());
+ }
+ Assert.assertTrue(operatorNames.contains("O1"));
+ Assert.assertTrue(operatorNames.contains("O2"));
+ Assert.assertTrue(operatorNames.contains(componentName("Ma", "M1", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Ma", "M2", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Ma", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Mb", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Mb", "M1", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Mb", "O2")));
+ Assert.assertTrue(operatorNames.contains(componentName("Mc", "M1", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Mc", "M2", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Mc", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Md", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Md", "M1", "O1")));
+ Assert.assertTrue(operatorNames.contains(componentName("Md", "O2")));
+
+ validateOperatorPropertyValue(dag, "O1", 1);
+ validateOperatorPropertyValue(dag, "O2", 2);
+ validateOperatorPropertyValue(dag, componentName("Ma", "M1", "O1"), 11);
+ validateOperatorPropertyValue(dag, componentName("Ma", "M2", "O1"), 12);
+ validateOperatorPropertyValue(dag, componentName("Ma", "O1"), 13);
+ validateOperatorPropertyValue(dag, componentName("Mb", "O1"), 21);
+ validateOperatorPropertyValue(dag, componentName("Mb", "M1", "O1"), 22);
+ validateOperatorPropertyValue(dag, componentName("Mb", "O2"), 23);
+ validateOperatorPropertyValue(dag, componentName("Mc", "M1", "O1"), 31);
+ validateOperatorPropertyValue(dag, componentName("Mc", "M2", "O1"), 32);
+ validateOperatorPropertyValue(dag, componentName("Mc", "O1"), 33);
+ validateOperatorPropertyValue(dag, componentName("Md", "O1"), 41);
+ validateOperatorPropertyValue(dag, componentName("Md", "M1", "O1"), 42);
+ validateOperatorPropertyValue(dag, componentName("Md", "O2"), 43);
+
+ validateOperatorParent(dag, "O1", null);
+ validateOperatorParent(dag, "O2", null);
+ validateOperatorParent(dag, componentName("Ma", "M1", "O1"), componentName("Ma", "M1"));
+ validateOperatorParent(dag, componentName("Ma", "M2", "O1"), componentName("Ma", "M2"));
+ validateOperatorParent(dag, componentName("Ma", "O1"), "Ma");
+ validateOperatorParent(dag, componentName("Mb", "O1"), "Mb");
+ validateOperatorParent(dag, componentName("Mb", "M1", "O1"), componentName("Mb", "M1"));
+ validateOperatorParent(dag, componentName("Mb", "O2"), "Mb");
+ validateOperatorParent(dag, componentName("Mc", "M1", "O1"), componentName("Mc", "M1"));
+ validateOperatorParent(dag, componentName("Mc", "M2", "O1"), componentName("Mc", "M2"));
+ validateOperatorParent(dag, componentName("Mc", "O1"), "Mc");
+ validateOperatorParent(dag, componentName("Md", "O1"), "Md");
+ validateOperatorParent(dag, componentName("Md", "M1", "O1"), componentName("Md", "M1"));
+ validateOperatorParent(dag, componentName("Md", "O2"), "Md");
+ }
+
+ private void validateOperatorParent(LogicalPlan dag, String operatorName, String parentModuleName)
+ {
+ LogicalPlan.OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
+ if (parentModuleName == null) {
+ Assert.assertNull(operatorMeta.getModuleName());
+ } else {
+ Assert.assertTrue(parentModuleName.equals(operatorMeta.getModuleName()));
+ }
+ }
+
+ private void validateOperatorPropertyValue(LogicalPlan dag, String operatorName, int expectedValue)
+ {
+ LogicalPlan.OperatorMeta oMeta = dag.getOperatorMeta(operatorName);
+ if (operatorName.equals("O1")) {
+ DummyInputOperator operator = (DummyInputOperator)oMeta.getOperator();
+ Assert.assertEquals(expectedValue, operator.getInputOperatorProp());
+ } else {
+ DummyOperator operator = (DummyOperator)oMeta.getOperator();
+ Assert.assertEquals(expectedValue, operator.getOperatorProp());
+ }
+ }
+
+ private void validatePublicMethods(LogicalPlan dag)
+ {
+ // Logical dag contains 4 modules added on top level.
+ List<String> moduleNames = new ArrayList<>();
+ for (LogicalPlan.ModuleMeta moduleMeta : dag.getAllModules()) {
+ moduleNames.add(moduleMeta.getName());
+ }
+ Assert.assertTrue(moduleNames.contains("Ma"));
+ Assert.assertTrue(moduleNames.contains("Mb"));
+ Assert.assertTrue(moduleNames.contains("Mc"));
+ Assert.assertTrue(moduleNames.contains("Md"));
+ Assert.assertTrue(moduleNames.contains("Me"));
+ Assert.assertEquals("Number of modules are 5", 5, dag.getAllModules().size());
+
+ // correct module meta is returned by getMeta call.
+ LogicalPlan.ModuleMeta m = dag.getModuleMeta("Ma");
+ Assert.assertEquals("Name of module is Ma", m.getName(), "Ma");
+
+ }
+
+ private static String componentName(String... names)
+ {
+ if (names.length == 0) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder(names[0]);
+ for (int i = 1; i < names.length; i++) {
+ sb.append(LogicalPlan.MODULE_NAMESPACE_SEPARATOR);
+ sb.append(names[i]);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Generate a conflict, Add a top level operator with name "m1_O1",
+ * and add a module "m1" which will populate operator "O1", causing name conflict with
+ * top level operator.
+ */
+ @Test(expected = java.lang.IllegalArgumentException.class)
+ public void conflictingNamesWithExpandedModule()
+ {
+ Configuration conf = new Configuration(false);
+ LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = new LogicalPlan();
+ DummyInputOperator in = dag.addOperator(componentName("m1", "O1"), new DummyInputOperator());
+ Level2ModuleA module = dag.addModule("m1", new Level2ModuleA());
+ dag.addStream("s1", in.out, module.mIn);
+ lpc.prepareDAG(dag, null, "ModuleApp");
+ dag.validate();
+ }
+
+ /**
+ * Module and Operator with same name is not allowed in a DAG, to prevent properties
+ * conflict.
+ */
+ @Test(expected = java.lang.IllegalArgumentException.class)
+ public void conflictingNamesWithOperator1()
+ {
+ Configuration conf = new Configuration(false);
+ LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = new LogicalPlan();
+ DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
+ Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
+ dag.addStream("s1", in.out, module.mIn);
+ lpc.prepareDAG(dag, null, "ModuleApp");
+ dag.validate();
+ }
+
+ /**
+ * Module and Operator with same name is not allowed in a DAG, to prevent properties
+ * conflict.
+ */
+ @Test(expected = java.lang.IllegalArgumentException.class)
+ public void conflictingNamesWithOperator2()
+ {
+ Configuration conf = new Configuration(false);
+ LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = new LogicalPlan();
+ Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
+ DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
+ dag.addStream("s1", in.out, module.mIn);
+ lpc.prepareDAG(dag, null, "ModuleApp");
+ dag.validate();
+ }
+}