You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by cs...@apache.org on 2015/09/17 06:58:54 UTC
[1/8] incubator-apex-core git commit: APEX-28 #resolve
Repository: incubator-apex-core
Updated Branches:
refs/heads/devel-3 7503dde51 -> 454feccac
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java b/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java
index 3b6bdd1..218156b 100644
--- a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java
@@ -15,28 +15,6 @@
*/
package com.datatorrent.stram.plan;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.lang.reflect.Field;
-import java.util.*;
-
-import javax.validation.ValidationException;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.mutable.MutableBoolean;
-import org.apache.hadoop.conf.Configuration;
-
import com.datatorrent.api.*;
import com.datatorrent.api.Attribute.AttributeMap.AttributeInitializer;
import com.datatorrent.api.Context.DAGContext;
@@ -46,20 +24,48 @@ import com.datatorrent.api.StringCodec.Integer2String;
import com.datatorrent.api.annotation.ApplicationAnnotation;
import com.datatorrent.common.codec.JsonStreamCodec;
import com.datatorrent.common.util.BasicContainerOptConfigurator;
+import com.datatorrent.common.util.FSStorageAgent;
import com.datatorrent.stram.PartitioningTest.PartitionLoadWatch;
import com.datatorrent.stram.client.StramClientUtils;
import com.datatorrent.stram.engine.GenericTestOperator;
import com.datatorrent.stram.engine.TestGeneratorInputOperator;
import com.datatorrent.stram.plan.LogicalPlanTest.ValidationTestOperator;
import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.AttributeParseUtils;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.ConfElement;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.ContextUtils;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.StramElement;
+import com.datatorrent.stram.plan.logical.MockStorageAgent;
import com.datatorrent.stram.support.StramTestSupport.RegexMatcher;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.lang.reflect.Field;
+import javax.validation.ValidationException;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.hadoop.conf.Configuration;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
import static org.junit.Assert.*;
+
+
public class LogicalPlanConfigurationTest {
private static OperatorMeta assertNode(LogicalPlan dag, String id) {
@@ -75,7 +81,6 @@ public class LogicalPlanConfigurationTest {
public void testLoadFromConfigXml() {
Configuration conf = new Configuration(false);
conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
- //Configuration.dumpConfiguration(conf, new PrintWriter(System.out));
LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
@@ -117,7 +122,7 @@ public class LogicalPlanConfigurationTest {
assertEquals("operator 2 number of outputs", 1, operator2.getOutputStreams().size());
StreamMeta fromNode2 = operator2.getOutputStreams().values().iterator().next();
- Set<OperatorMeta> targetNodes = new HashSet<OperatorMeta>();
+ Set<OperatorMeta> targetNodes = Sets.newHashSet();
for (LogicalPlan.InputPortMeta ip : fromNode2.getSinks()) {
targetNodes.add(ip.getOperatorWrapper());
}
@@ -191,7 +196,7 @@ public class LogicalPlanConfigurationTest {
StreamMeta input1 = dag.getStream("inputStream");
assertNotNull(input1);
Assert.assertEquals("input1 source", dag.getOperatorMeta("inputOperator"), input1.getSource().getOperatorMeta());
- Set<OperatorMeta> targetNodes = new HashSet<OperatorMeta>();
+ Set<OperatorMeta> targetNodes = Sets.newHashSet();
for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
targetNodes.add(targetPort.getOperatorWrapper());
}
@@ -221,11 +226,11 @@ public class LogicalPlanConfigurationTest {
dag.validate();
assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
- Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = new HashMap<Class<?>, Class<? extends StringCodec<?>>>();
+ Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
stringCodecsMap.put(Integer.class, Integer2String.class);
assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());
-
+
assertEquals("number of operator confs", 5, dag.getAllOperators().size());
assertEquals("number of root operators", 1, dag.getRootOperators().size());
@@ -241,7 +246,7 @@ public class LogicalPlanConfigurationTest {
for(OutputPortMeta opm : input.getOutputStreams().keySet()){
assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
}
-
+
OperatorMeta operator3 = dag.getOperatorMeta("operator3");
assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
@@ -259,7 +264,7 @@ public class LogicalPlanConfigurationTest {
assertNotNull(input1);
OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
- Set<OperatorMeta> targetNodes = new HashSet<OperatorMeta>();
+ Set<OperatorMeta> targetNodes = Sets.newHashSet();
for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
targetNodes.add(targetPort.getOperatorWrapper());
}
@@ -269,6 +274,7 @@ public class LogicalPlanConfigurationTest {
}
@Test
+ @SuppressWarnings("UnnecessaryBoxing")
public void testAppLevelAttributes()
{
String appName = "app1";
@@ -296,6 +302,7 @@ public class LogicalPlanConfigurationTest {
}
@Test
+ @SuppressWarnings("UnnecessaryBoxing")
public void testAppLevelProperties() {
String appName ="app1";
Properties props =new Properties();
@@ -315,6 +322,7 @@ public class LogicalPlanConfigurationTest {
Assert.assertEquals("",Integer.valueOf(1000),app1Test.getTestprop3());
Assert.assertEquals("",Integer.valueOf(10000),app1Test.getInncls().getA());
}
+
@Test
public void testPrepareDAG() {
final MutableBoolean appInitialized = new MutableBoolean(false);
@@ -365,6 +373,7 @@ public class LogicalPlanConfigurationTest {
Operator operator3 = dag.addOperator("operator3", new GenericTestOperator());
LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false));
+ LOG.debug("calling addFromProperties");
pb.addFromProperties(props, null);
Map<String, String> configProps = pb.getProperties(dag.getMeta(operator1), "appName");
@@ -415,7 +424,6 @@ public class LogicalPlanConfigurationTest {
@Override
public void populateDAG(DAG dag, Configuration conf)
{
- //dag.setAttribute(DAGContext.APPLICATION_NAME, "testApp");
}
}
@@ -478,6 +486,7 @@ public class LogicalPlanConfigurationTest {
}
@Test
+ @SuppressWarnings( {"UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes"})
public void testOperatorLevelAttributes() {
String appName = "app1";
StreamingApplication app = new StreamingApplication() {
@@ -581,22 +590,10 @@ public class LogicalPlanConfigurationTest {
}
@Test
+ @SuppressWarnings("UnnecessaryBoxing")
public void testPortLevelAttributes() {
String appName = "app1";
- final GenericTestOperator gt1 = new GenericTestOperator();
- final GenericTestOperator gt2 = new GenericTestOperator();
- final GenericTestOperator gt3 = new GenericTestOperator();
- StreamingApplication app = new StreamingApplication() {
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- dag.addOperator("operator1", gt1);
- dag.addOperator("operator2", gt2);
- dag.addOperator("operator3", gt3);
- dag.addStream("s1", gt1.outport1, gt2.inport1);
- dag.addStream("s2", gt2.outport1, gt3.inport1, gt3.inport2);
- }
- };
+ SimpleTestApplication app = new SimpleTestApplication();
Properties props = new Properties();
props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
@@ -613,33 +610,31 @@ public class LogicalPlanConfigurationTest {
LogicalPlan dag = new LogicalPlan();
dagBuilder.prepareDAG(dag, app, appPath);
- //dagBuilder.populateDAG(dag, new Configuration(false));
- //dagBuilder.setApplicationConfiguration(dag, appName);
OperatorMeta om1 = dag.getOperatorMeta("operator1");
- Assert.assertEquals("", Integer.valueOf(16 * 1024), om1.getMeta(gt1.outport1).getValue(PortContext.QUEUE_CAPACITY));
+ Assert.assertEquals("", Integer.valueOf(16 * 1024), om1.getMeta(app.gt1.outport1).getValue(PortContext.QUEUE_CAPACITY));
OperatorMeta om2 = dag.getOperatorMeta("operator2");
- Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(gt2.inport1).getValue(PortContext.QUEUE_CAPACITY));
- Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(gt2.outport1).getValue(PortContext.QUEUE_CAPACITY));
+ Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(app.gt2.inport1).getValue(PortContext.QUEUE_CAPACITY));
+ Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(app.gt2.outport1).getValue(PortContext.QUEUE_CAPACITY));
OperatorMeta om3 = dag.getOperatorMeta("operator3");
- Assert.assertEquals("", Integer.valueOf(16 * 1024), om3.getMeta(gt3.inport1).getValue(PortContext.QUEUE_CAPACITY));
- Assert.assertEquals("", Integer.valueOf(32 * 1024), om3.getMeta(gt3.inport2).getValue(PortContext.QUEUE_CAPACITY));
+ Assert.assertEquals("", Integer.valueOf(16 * 1024), om3.getMeta(app.gt3.inport1).getValue(PortContext.QUEUE_CAPACITY));
+ Assert.assertEquals("", Integer.valueOf(32 * 1024), om3.getMeta(app.gt3.inport2).getValue(PortContext.QUEUE_CAPACITY));
}
@Test
public void testInvalidAttribute() throws Exception {
-
Assert.assertNotSame(0, com.datatorrent.api.Context.DAGContext.serialVersionUID);
- Set<Attribute<Object>> appAttributes = AttributeInitializer.getAttributes(com.datatorrent.api.Context.DAGContext.class);
- Attribute<Object> attribute = new Attribute<Object>("", null);
+ Attribute<String> attribute = new Attribute<>("", null);
Field nameField = Attribute.class.getDeclaredField("name");
nameField.setAccessible(true);
nameField.set(attribute, "NOT_CONFIGURABLE");
nameField.setAccessible(false);
- appAttributes.add(attribute);
+ ContextUtils.addAttribute(com.datatorrent.api.Context.DAGContext.class, attribute);
+ AttributeParseUtils.initialize();
+ ConfElement.initialize();
// attribute that cannot be configured
@@ -656,7 +651,9 @@ public class LogicalPlanConfigurationTest {
Assert.assertThat("Attribute not configurable", e.getMessage(), RegexMatcher.matches("Attribute does not support property configuration: NOT_CONFIGURABLE.*"));
}
- appAttributes.remove(attribute);
+ ContextUtils.removeAttribute(com.datatorrent.api.Context.DAGContext.class, attribute);
+ AttributeParseUtils.initialize();
+ ConfElement.initialize();
// invalid attribute name
props = new Properties();
@@ -667,9 +664,9 @@ public class LogicalPlanConfigurationTest {
new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
Assert.fail("Exception expected");
} catch (Exception e) {
+ LOG.debug("Exception message: {}", e.getMessage());
Assert.assertThat("Invalid attribute name", e.getMessage(), RegexMatcher.matches("Invalid attribute reference: " + invalidAttribute));
}
-
}
@Test
@@ -759,6 +756,898 @@ public class LogicalPlanConfigurationTest {
dag.validate();
}
+ /**
+ * This test and all of the following ambiguous attribute tests verify that when an ambiguous attribute
+ * name is provided, all the corresponding attributes are set.
+ * <br/><br/>
+ * <b>Note:</b> Ambiguous attribute means that when multiple attributes with the same
+ * simple name exist for multiple types of dag elements (like operators and ports).
+ * An example of such attributes are the com.datatorrent.api.Context.OperatorContext.AUTO_RECORD
+ * and com.datatorrent.api.Context.PortContext.AUTO_RECORD.
+ * <br/><br/>
+ * This test should set the attribute on the operators and ports.
+ */
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testRootLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX,
+ null,
+ Boolean.TRUE,
+ true,
+ true);
+ }
+
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testApplicationLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ null,
+ Boolean.TRUE,
+ true,
+ true);
+ }
+
+ /**
+ * This should only set the attribute on the operator
+ */
+ @Test
+ public void testOperatorLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ null,
+ Boolean.TRUE,
+ true,
+ false);
+ }
+
+ /**
+ * This should only set the attribute on the port
+ */
+ @Test
+ public void testPortLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX
+ + "port"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ null,
+ Boolean.TRUE,
+ false,
+ true);
+ }
+
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testRootLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX,
+ PortContext.class.getCanonicalName(),
+ Boolean.TRUE,
+ false,
+ true);
+ }
+
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testApplicationLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ PortContext.class.getCanonicalName(),
+ Boolean.TRUE,
+ false,
+ true);
+ }
+
+ /**
+ * This should only set the attribute on the operator
+ */
+ @Test
+ public void testOperatorLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ OperatorContext.class.getCanonicalName(),
+ Boolean.TRUE,
+ true,
+ false);
+ }
+
+ /**
+ * This should only set the attribute on the port
+ */
+ @Test
+ public void testOperatorLevelAmbiguousAttributeComplex2()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ PortContext.class.getCanonicalName(),
+ Boolean.TRUE,
+ false,
+ true);
+ }
+
+ /**
+ * This should only set the attribute on the port
+ */
+ @Test
+ public void testPortLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
+ Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX
+ + "port"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ PortContext.class.getCanonicalName(),
+ Boolean.TRUE,
+ false,
+ true);
+ }
+
+ private void testAttributeAmbiguousSimpleHelper(Attribute<?> attributeObjOperator,
+ Attribute<?> attributeObjPort,
+ String root,
+ String contextClass,
+ Object val,
+ boolean operatorSet,
+ boolean portSet)
+ {
+ Properties props = propertiesBuilder(attributeObjOperator.getSimpleName(),
+ root,
+ contextClass,
+ val);
+
+ simpleAttributeOperatorHelperAssert(attributeObjOperator,
+ props,
+ val,
+ operatorSet);
+
+ simpleNamePortAssertHelperAssert(attributeObjPort,
+ props,
+ val,
+ portSet);
+ }
+
+ @Test
+ public void testRootLevelAttributeSimpleNameOperator()
+ {
+ simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB,
+ StreamingApplication.DT_PREFIX,
+ true,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ @Test
+ public void testRootLevelStorageAgentSimpleNameOperator()
+ {
+ MockStorageAgent mockAgent = new MockStorageAgent();
+
+ simpleAttributeOperatorHelper(OperatorContext.STORAGE_AGENT,
+ StreamingApplication.DT_PREFIX,
+ true,
+ mockAgent,
+ true,
+ false);
+ }
+
+ @Test
+ public void testRootLevelAttributeSimpleNameOperatorNoScope()
+ {
+ simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB,
+ StreamingApplication.DT_PREFIX,
+ true,
+ (Integer)4096,
+ true,
+ false);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNameOperator()
+ {
+ simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "SimpleTestApp"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ private void simpleAttributeOperatorHelper(Attribute<?> attributeObj,
+ String root,
+ boolean simpleName,
+ Object val,
+ boolean set,
+ boolean scope)
+ {
+ Properties props = propertiesBuilderOperator(attributeObj.getSimpleName(),
+ root,
+ simpleName,
+ val,
+ scope);
+
+ simpleAttributeOperatorHelperAssert(attributeObj,
+ props,
+ val,
+ set);
+ }
+
+ private void simpleAttributeOperatorHelperAssert(Attribute<?> attributeObj,
+ Properties props,
+ Object val,
+ boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ OperatorMeta om1 = dag.getOperatorMeta("operator1");
+
+ if (set) {
+ Assert.assertEquals(val, om1.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(val, om1.getValue(attributeObj));
+ }
+
+ OperatorMeta om2 = dag.getOperatorMeta("operator2");
+
+ if (set) {
+ Assert.assertEquals(val, om2.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(val, om2.getValue(attributeObj));
+ }
+
+ OperatorMeta om3 = dag.getOperatorMeta("operator3");
+
+ if (set) {
+ Assert.assertEquals(val, om3.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(val, om3.getValue(attributeObj));
+ }
+ }
+
+ /* Port tests */
+ @Test
+ public void testRootLevelAttributeSimpleNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ true,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ @Test
+ public void testRootLevelAttributeSimpleNamePortNoScope()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ true,
+ (Integer)4096,
+ true,
+ false);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeSimpleNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "SimpleTestApp"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ false,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNamePortNoScope()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ false,
+ (Integer)4096,
+ true,
+ false);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeComplexNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeComplexNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "SimpleTestApp"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false,
+ (Integer)4096,
+ true,
+ true);
+ }
+
+ /* Input port tests */
+ @Test
+ public void testRootLevelAttributeSimpleNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ true,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeSimpleNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "SimpleTestApp"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ false,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeComplexNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeComplexNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "SimpleTestApp"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false,
+ (Integer)4096,
+ true);
+ }
+
+ /* Output port tests */
+ @Test
+ public void testRootLevelAttributeSimpleNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ true,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeSimpleNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "SimpleTestApp"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX,
+ false,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeComplexNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false,
+ (Integer)4096,
+ true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeComplexNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
+ StreamingApplication.DT_PREFIX
+ + "application"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "SimpleTestApp"
+ + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false,
+ (Integer)4096,
+ true);
+ }
+
+ /* Helpers for building ports */
+ private void simpleAttributePortHelper(Attribute<?> attributeObj,
+ String root,
+ boolean simpleName,
+ Object val,
+ boolean set,
+ boolean scope)
+ {
+ Properties props = propertiesBuilderPort(attributeObj.getSimpleName(),
+ root,
+ simpleName,
+ val,
+ scope);
+
+ simpleNamePortAssertHelperAssert(attributeObj,
+ props,
+ val,
+ set);
+ }
+
+ private void simpleAttributeInputPortHelper(Attribute<?> attributeObj,
+ String root,
+ boolean simpleName,
+ Object val,
+ boolean set)
+ {
+ Properties props = propertiesBuilderInputPort(attributeObj.getSimpleName(),
+ root,
+ simpleName,
+ val);
+
+ simpleNameInputPortAssertHelperAssert(attributeObj,
+ props,
+ val,
+ set);
+
+ simpleNameOutputPortAssertHelperAssert(attributeObj,
+ props,
+ val,
+ !set);
+ }
+
+ private void simpleAttributeOutputPortHelper(Attribute<?> attributeObj,
+ String root,
+ boolean simpleName,
+ Object val,
+ boolean set)
+ {
+ Properties props = propertiesBuilderOutputPort(attributeObj.getSimpleName(),
+ root,
+ simpleName,
+ val);
+
+ simpleNameOutputPortAssertHelperAssert(attributeObj,
+ props,
+ val,
+ set);
+
+ simpleNameInputPortAssertHelperAssert(attributeObj,
+ props,
+ val,
+ !set);
+ }
+
+ private void simpleNamePortAssertHelperAssert(Attribute<?> attributeObj,
+ Properties props,
+ Object val,
+ boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ simpleNamePortAssertHelper(attributeObj,
+ dag,
+ "operator1",
+ val,
+ set);
+
+ simpleNamePortAssertHelper(attributeObj,
+ dag,
+ "operator2",
+ val,
+ set);
+
+ simpleNamePortAssertHelper(attributeObj,
+ dag,
+ "operator3",
+ val,
+ set);
+ }
+
+ private void simpleNameInputPortAssertHelperAssert(Attribute<?> attributeObj,
+ Properties props,
+ Object val,
+ boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ simpleNameInputPortAssertHelper(attributeObj,
+ dag,
+ "operator1",
+ val,
+ set);
+
+ simpleNameInputPortAssertHelper(attributeObj,
+ dag,
+ "operator2",
+ val,
+ set);
+
+ simpleNameInputPortAssertHelper(attributeObj,
+ dag,
+ "operator3",
+ val,
+ set);
+ }
+
+ private void simpleNameOutputPortAssertHelperAssert(Attribute<?> attributeObj,
+ Properties props,
+ Object val,
+ boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ simpleNameOutputPortAssertHelper(attributeObj,
+ dag,
+ "operator1",
+ val,
+ set);
+
+ simpleNameOutputPortAssertHelper(attributeObj,
+ dag,
+ "operator2",
+ val,
+ set);
+
+ simpleNameOutputPortAssertHelper(attributeObj,
+ dag,
+ "operator3",
+ val,
+ set);
+ }
+
+ private void simpleNamePortAssertHelper(Attribute<?> attributeObj,
+ LogicalPlan dag,
+ String operatorName,
+ Object queueCapacity,
+ boolean set)
+ {
+ simpleNameInputPortAssertHelper(attributeObj,
+ dag,
+ operatorName,
+ queueCapacity,
+ set);
+
+ simpleNameOutputPortAssertHelper(attributeObj,
+ dag,
+ operatorName,
+ queueCapacity,
+ set);
+ }
+
+ private void simpleNameInputPortAssertHelper(Attribute<?> attributeObj,
+ LogicalPlan dag,
+ String operatorName,
+ Object queueCapacity,
+ boolean set)
+ {
+ OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
+
+ for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
+ if (set) {
+ Assert.assertEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
+ }
+ }
+ }
+
+ private void simpleNameOutputPortAssertHelper(Attribute<?> attributeObj,
+ LogicalPlan dag,
+ String operatorName,
+ Object queueCapacity,
+ boolean set)
+ {
+ OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
+
+ for (OutputPortMeta outputPortMeta: operatorMeta.getOutputStreams().keySet()) {
+ if (set) {
+ Assert.assertEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
+ }
+ }
+ }
+
+ /* Helpers for building properties */
+ private Properties propertiesBuilder(String attributeName,
+ String root,
+ String contextClass,
+ Object val)
+ {
+ boolean simpleName = contextClass == null;
+
+ if (!simpleName) {
+ attributeName = contextClass
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + attributeName;
+ }
+
+ Properties props = new Properties();
+
+ String propName = root
+ + StramElement.ATTR.getValue()
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + attributeName;
+
+ LOG.debug("adding prop {} with value {}", propName, val.toString());
+
+ props.put(propName,
+ val.toString());
+
+ return props;
+ }
+
+ private Properties propertiesBuilderOperator(String attributeName,
+ String root,
+ boolean simpleName,
+ Object val,
+ boolean addOperator)
+ {
+ String contextClass = simpleName ? null : OperatorContext.class.getCanonicalName();
+
+ if (addOperator) {
+ root += "operator"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR;
+ }
+
+ return propertiesBuilder(attributeName,
+ root,
+ contextClass,
+ val);
+ }
+
+ private Properties propertiesBuilderPort(String attributeName,
+ String root,
+ boolean simpleName,
+ Object val,
+ boolean addPort)
+ {
+ String contextClass = simpleName ? null : PortContext.class.getCanonicalName();
+
+ if (addPort) {
+ root += "port"
+ + LogicalPlanConfiguration.KEY_SEPARATOR
+ + "*"
+ + LogicalPlanConfiguration.KEY_SEPARATOR;
+ }
+
+ return propertiesBuilder(attributeName,
+ root,
+ contextClass,
+ val);
+ }
+
+ private Properties propertiesBuilderInputPort(String attributeName,
+ String root,
+ boolean simpleName,
+ Object val)
+ {
+ String contextClass = simpleName ? null: PortContext.class.getCanonicalName();
+
+ root += "inputport" +
+ LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" +
+ LogicalPlanConfiguration.KEY_SEPARATOR;
+
+ return propertiesBuilder(attributeName,
+ root,
+ contextClass,
+ val);
+ }
+
+ private Properties propertiesBuilderOutputPort(String attributeName,
+ String root,
+ boolean simpleName,
+ Object val)
+ {
+ String contextClass = simpleName ? null: PortContext.class.getCanonicalName();
+
+ root += "outputport" +
+ LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" +
+ LogicalPlanConfiguration.KEY_SEPARATOR;
+
+ return propertiesBuilder(attributeName,
+ root,
+ contextClass,
+ val);
+ }
+
private static final Logger logger = LoggerFactory.getLogger(LogicalPlanConfigurationTest.class);
public static class TestApplication implements StreamingApplication {
@@ -818,11 +1707,11 @@ public class LogicalPlanConfigurationTest {
}
}
}
-
+
public static class TestStatsListener implements StatsListener{
-
+
private int intProp;
-
+
public TestStatsListener()
{
}
@@ -866,11 +1755,34 @@ public class LogicalPlanConfigurationTest {
return false;
return true;
}
-
}
public static class TestSchema
{
}
+
+ public static class SimpleTestApplication implements StreamingApplication
+ {
+ public final GenericTestOperator gt1 = new GenericTestOperator();
+ public final GenericTestOperator gt2 = new GenericTestOperator();
+ public final GenericTestOperator gt3 = new GenericTestOperator();
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ dag.addOperator("operator1", gt1);
+ dag.addOperator("operator2", gt2);
+ dag.addOperator("operator3", gt3);
+ dag.addStream("s1", gt1.outport1, gt2.inport1);
+ dag.addStream("s2", gt2.outport1, gt3.inport1, gt3.inport2);
+ }
+ };
+
+ @ApplicationAnnotation(name="SimpleTestApp")
+ public static class SimpleTestApplicationWithName extends SimpleTestApplication
+ {
+ };
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogicalPlanConfigurationTest.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java b/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java
index ac05bed..78173d8 100644
--- a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java
@@ -19,6 +19,7 @@ import com.datatorrent.common.util.BaseOperator;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Serializable;
+
import java.util.*;
import javax.validation.*;
@@ -50,6 +51,7 @@ import com.datatorrent.stram.engine.TestGeneratorInputOperator;
import com.datatorrent.stram.engine.TestNonOptionalOutportInputOperator;
import com.datatorrent.stram.engine.TestOutputOperator;
import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
import com.datatorrent.stram.support.StramTestSupport.MemoryStorageAgent;
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/engine/src/test/java/com/datatorrent/stram/plan/logical/MockStorageAgent.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/MockStorageAgent.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/MockStorageAgent.java
new file mode 100644
index 0000000..3975e02
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/MockStorageAgent.java
@@ -0,0 +1,67 @@
+/**
+ * Copyright (C) 2015 DataTorrent, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.datatorrent.stram.plan.logical;
+
+import com.datatorrent.api.StorageAgent;
+import java.io.IOException;
+
+public class MockStorageAgent implements StorageAgent
+{
+ public MockStorageAgent()
+ {
+ }
+
+ @Override
+ public void save(Object object, int operatorId, long windowId) throws IOException
+ {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public Object load(int operatorId, long windowId) throws IOException
+ {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public void delete(int operatorId, long windowId) throws IOException
+ {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public long[] getWindowIds(int operatorId) throws IOException
+ {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public String toString()
+ {
+ return MockStorageAgent.class.getCanonicalName();
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if(obj == null) {
+ return false;
+ }
+
+ return obj instanceof MockStorageAgent;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/engine/src/test/resources/testTopology.json
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testTopology.json b/engine/src/test/resources/testTopology.json
index 1ea9756..62c5262 100644
--- a/engine/src/test/resources/testTopology.json
+++ b/engine/src/test/resources/testTopology.json
@@ -13,7 +13,7 @@
"STATS_LISTENERS" : {
"java.util.ArrayList" : [
{
- "com.datatorrent.stram.plan.LogicalPlanConfigurationTest$TestStatsListener" : {
+ "com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest$TestStatsListener" : {
"intProp" : 222
}
}
@@ -38,7 +38,7 @@
"com.datatorrent.stram.engine.GenericTestOperator":{
"myStringProperty": "myStringPropertyValue"
}
-
+
}
},
{
[6/8] incubator-apex-core git commit: APEX-28 #resolve
Posted by cs...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java b/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java
deleted file mode 100644
index 218156b..0000000
--- a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanConfigurationTest.java
+++ /dev/null
@@ -1,1788 +0,0 @@
-/**
- * Copyright (C) 2015 DataTorrent, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.datatorrent.stram.plan;
-
-import com.datatorrent.api.*;
-import com.datatorrent.api.Attribute.AttributeMap.AttributeInitializer;
-import com.datatorrent.api.Context.DAGContext;
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.Context.PortContext;
-import com.datatorrent.api.StringCodec.Integer2String;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.codec.JsonStreamCodec;
-import com.datatorrent.common.util.BasicContainerOptConfigurator;
-import com.datatorrent.common.util.FSStorageAgent;
-import com.datatorrent.stram.PartitioningTest.PartitionLoadWatch;
-import com.datatorrent.stram.client.StramClientUtils;
-import com.datatorrent.stram.engine.GenericTestOperator;
-import com.datatorrent.stram.engine.TestGeneratorInputOperator;
-import com.datatorrent.stram.plan.LogicalPlanTest.ValidationTestOperator;
-import com.datatorrent.stram.plan.logical.LogicalPlan;
-import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
-import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
-import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
-import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
-import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
-import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.AttributeParseUtils;
-import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.ConfElement;
-import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.ContextUtils;
-import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.StramElement;
-import com.datatorrent.stram.plan.logical.MockStorageAgent;
-import com.datatorrent.stram.support.StramTestSupport.RegexMatcher;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.lang.reflect.Field;
-import javax.validation.ValidationException;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.mutable.MutableBoolean;
-import org.apache.hadoop.conf.Configuration;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-import static org.junit.Assert.*;
-
-
-
-public class LogicalPlanConfigurationTest {
-
- private static OperatorMeta assertNode(LogicalPlan dag, String id) {
- OperatorMeta n = dag.getOperatorMeta(id);
- assertNotNull("operator exists id=" + id, n);
- return n;
- }
-
- /**
- * Test read from dt-site.xml in Hadoop configuration format.
- */
- @Test
- public void testLoadFromConfigXml() {
- Configuration conf = new Configuration(false);
- conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
-
- LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
-
- LogicalPlan dag = new LogicalPlan();
- builder.populateDAG(dag);
- dag.validate();
-
- assertEquals("number of operator confs", 6, dag.getAllOperators().size());
-
- OperatorMeta operator1 = assertNode(dag, "operator1");
- OperatorMeta operator2 = assertNode(dag, "operator2");
- OperatorMeta operator3 = assertNode(dag, "operator3");
- OperatorMeta operator4 = assertNode(dag, "operator4");
-
- assertNotNull("operatorConf for root", operator1);
- assertEquals("operatorId set", "operator1", operator1.getName());
-
- // verify operator instantiation
- assertEquals(operator1.getOperator().getClass(), TestGeneratorInputOperator.class);
- TestGeneratorInputOperator GenericTestNode = (TestGeneratorInputOperator)operator1.getOperator();
- assertEquals("myStringPropertyValue", GenericTestNode.getMyStringProperty());
-
- // check links
- assertEquals("operator1 inputs", 0, operator1.getInputStreams().size());
- assertEquals("operator1 outputs", 1, operator1.getOutputStreams().size());
- StreamMeta n1n2 = operator2.getInputStreams().get(operator2.getMeta(((GenericTestOperator)operator2.getOperator()).inport1));
- assertNotNull("n1n2", n1n2);
-
- // output/input stream object same
- assertEquals("rootNode out is operator2 in", n1n2, operator1.getOutputStreams().get(operator1.getMeta(((TestGeneratorInputOperator)operator1.getOperator()).outport)));
- assertEquals("n1n2 source", operator1, n1n2.getSource().getOperatorMeta());
- Assert.assertEquals("n1n2 targets", 1, n1n2.getSinks().size());
- Assert.assertEquals("n1n2 target", operator2, n1n2.getSinks().get(0).getOperatorWrapper());
-
- assertEquals("stream name", "n1n2", n1n2.getName());
- Assert.assertEquals("n1n2 not inline (default)", null, n1n2.getLocality());
-
- // operator 2 streams to operator 3 and operator 4
- assertEquals("operator 2 number of outputs", 1, operator2.getOutputStreams().size());
- StreamMeta fromNode2 = operator2.getOutputStreams().values().iterator().next();
-
- Set<OperatorMeta> targetNodes = Sets.newHashSet();
- for (LogicalPlan.InputPortMeta ip : fromNode2.getSinks()) {
- targetNodes.add(ip.getOperatorWrapper());
- }
- Assert.assertEquals("outputs " + fromNode2, Sets.newHashSet(operator3, operator4), targetNodes);
-
- OperatorMeta operator6 = assertNode(dag, "operator6");
-
- List<OperatorMeta> rootNodes = dag.getRootOperators();
- assertEquals("number root operators", 2, rootNodes.size());
- assertTrue("root operator2", rootNodes.contains(operator1));
- assertTrue("root operator6", rootNodes.contains(operator6));
-
- for (OperatorMeta n : rootNodes) {
- printTopology(n, dag, 0);
- }
-
- }
-
- private void printTopology(OperatorMeta operator, DAG tplg, int level) {
- String prefix = "";
- if (level > 0) {
- prefix = StringUtils.repeat(" ", 20*(level-1)) + " |" + StringUtils.repeat("-", 17);
- }
- logger.debug(prefix + operator.getName());
- for (StreamMeta downStream : operator.getOutputStreams().values()) {
- if (!downStream.getSinks().isEmpty()) {
- for (LogicalPlan.InputPortMeta targetNode : downStream.getSinks()) {
- printTopology(targetNode.getOperatorWrapper(), tplg, level+1);
- }
- }
- }
- }
-
- @Test
- public void testLoadFromPropertiesFile() throws IOException
- {
- Properties props = new Properties();
- String resourcePath = "/testTopology.properties";
- InputStream is = this.getClass().getResourceAsStream(resourcePath);
- if (is == null) {
- fail("Could not load " + resourcePath);
- }
- props.load(is);
- LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false))
- .addFromProperties(props, null);
-
- LogicalPlan dag = new LogicalPlan();
- pb.populateDAG(dag);
- dag.validate();
-
- assertEquals("number of operator confs", 5, dag.getAllOperators().size());
- assertEquals("number of root operators", 1, dag.getRootOperators().size());
-
- StreamMeta s1 = dag.getStream("n1n2");
- assertNotNull(s1);
- assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
-
- OperatorMeta operator3 = dag.getOperatorMeta("operator3");
- assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
-
- GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
- assertEquals("myStringProperty " + doperator3, "myStringPropertyValueFromTemplate", doperator3.getMyStringProperty());
- assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
-
- OperatorMeta operator4 = dag.getOperatorMeta("operator4");
- GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
- assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
- assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
- assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
-
- StreamMeta input1 = dag.getStream("inputStream");
- assertNotNull(input1);
- Assert.assertEquals("input1 source", dag.getOperatorMeta("inputOperator"), input1.getSource().getOperatorMeta());
- Set<OperatorMeta> targetNodes = Sets.newHashSet();
- for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
- targetNodes.add(targetPort.getOperatorWrapper());
- }
-
- Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
-
- }
-
- @Test
- public void testLoadFromJson() throws Exception
- {
- String resourcePath = "/testTopology.json";
- InputStream is = this.getClass().getResourceAsStream(resourcePath);
- if (is == null) {
- fail("Could not load " + resourcePath);
- }
- StringWriter writer = new StringWriter();
-
- IOUtils.copy(is, writer);
- JSONObject json = new JSONObject(writer.toString());
-
- Configuration conf = new Configuration(false);
- conf.set(StreamingApplication.DT_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");
-
- LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
- LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
- dag.validate();
-
- assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
- Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
- stringCodecsMap.put(Integer.class, Integer2String.class);
- assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
- assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());
-
- assertEquals("number of operator confs", 5, dag.getAllOperators().size());
- assertEquals("number of root operators", 1, dag.getRootOperators().size());
-
- StreamMeta s1 = dag.getStream("n1n2");
- assertNotNull(s1);
- assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
-
- OperatorMeta input = dag.getOperatorMeta("inputOperator");
- TestStatsListener tsl = new TestStatsListener();
- tsl.setIntProp(222);
- List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
- assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
- for(OutputPortMeta opm : input.getOutputStreams().keySet()){
- assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
- }
-
- OperatorMeta operator3 = dag.getOperatorMeta("operator3");
- assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
-
- GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
- assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
- assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
-
- OperatorMeta operator4 = dag.getOperatorMeta("operator4");
- GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
- assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
- assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
- assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
-
- StreamMeta input1 = dag.getStream("inputStream");
- assertNotNull(input1);
- OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
- Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
- Set<OperatorMeta> targetNodes = Sets.newHashSet();
- for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
- targetNodes.add(targetPort.getOperatorWrapper());
- }
- Assert.assertEquals("operator attribute " + inputOperator, 64, (int)inputOperator.getValue(OperatorContext.MEMORY_MB));
- Assert.assertEquals("port attribute " + inputOperator, 8, (int)input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
- Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
- }
-
- @Test
- @SuppressWarnings("UnnecessaryBoxing")
- public void testAppLevelAttributes()
- {
- String appName = "app1";
-
- Properties props = new Properties();
- props.put(StreamingApplication.DT_PREFIX + DAG.MASTER_MEMORY_MB.getName(), "123");
- props.put(StreamingApplication.DT_PREFIX + DAG.CONTAINER_JVM_OPTIONS.getName(), "-Dlog4j.properties=custom_log4j.properties");
- props.put(StreamingApplication.DT_PREFIX + DAG.APPLICATION_PATH.getName(), "/defaultdir");
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + "." + DAG.APPLICATION_PATH.getName(), "/otherdir");
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + "." + DAG.STREAMING_WINDOW_SIZE_MILLIS.getName(), "1000");
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- LogicalPlan dag = new LogicalPlan();
-
- dagBuilder.populateDAG(dag);
-
- dagBuilder.setApplicationConfiguration(dag, appName, null);
-
- Assert.assertEquals("", "/otherdir", dag.getValue(DAG.APPLICATION_PATH));
- Assert.assertEquals("", Integer.valueOf(123), dag.getValue(DAG.MASTER_MEMORY_MB));
- Assert.assertEquals("", Integer.valueOf(1000), dag.getValue(DAG.STREAMING_WINDOW_SIZE_MILLIS));
- Assert.assertEquals("", "-Dlog4j.properties=custom_log4j.properties", dag.getValue(DAG.CONTAINER_JVM_OPTIONS));
-
- }
- @Test
- @SuppressWarnings("UnnecessaryBoxing")
- public void testAppLevelProperties() {
- String appName ="app1";
- Properties props =new Properties();
- props.put(StreamingApplication.DT_PREFIX + "application."+appName+".testprop1","10");
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".prop.testprop2", "100");
- props.put(StreamingApplication.DT_PREFIX + "application.*.prop.testprop3","1000");
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".inncls.a", "10000");
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- LogicalPlan dag = new LogicalPlan();
- TestApplication app1Test=new TestApplication();
-
- dagBuilder.setApplicationConfiguration(dag, appName,app1Test);
- Assert.assertEquals("",Integer.valueOf(10),app1Test.getTestprop1());
- Assert.assertEquals("",Integer.valueOf(100),app1Test.getTestprop2());
- Assert.assertEquals("",Integer.valueOf(1000),app1Test.getTestprop3());
- Assert.assertEquals("",Integer.valueOf(10000),app1Test.getInncls().getA());
- }
-
- @Test
- public void testPrepareDAG() {
- final MutableBoolean appInitialized = new MutableBoolean(false);
- StreamingApplication app = new StreamingApplication() {
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- Assert.assertEquals("", "hostname:9090", dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
- dag.setAttribute(DAG.GATEWAY_CONNECT_ADDRESS, "hostname:9091");
- appInitialized.setValue(true);
- }
- };
- Configuration conf = new Configuration(false);
- conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
- LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
-
- LogicalPlan dag = new LogicalPlan();
- pb.prepareDAG(dag, app, "testconfig");
-
- Assert.assertTrue("populateDAG called", appInitialized.booleanValue());
- Assert.assertEquals("populateDAG overrides attribute", "hostname:9091", dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
- }
-
- @Test
- public void testOperatorConfigurationLookup() {
-
- Properties props = new Properties();
-
- // match operator by name
- props.put(StreamingApplication.DT_PREFIX + "template.matchId1.matchIdRegExp", ".*operator1.*");
- props.put(StreamingApplication.DT_PREFIX + "template.matchId1.stringProperty2", "stringProperty2Value-matchId1");
- props.put(StreamingApplication.DT_PREFIX + "template.matchId1.nested.property", "nested.propertyValue-matchId1");
-
- // match class name, lower priority
- props.put(StreamingApplication.DT_PREFIX + "template.matchClass1.matchClassNameRegExp", ".*" + ValidationTestOperator.class.getSimpleName());
- props.put(StreamingApplication.DT_PREFIX + "template.matchClass1.stringProperty2", "stringProperty2Value-matchClass1");
-
- // match class name
- props.put(StreamingApplication.DT_PREFIX + "template.t2.matchClassNameRegExp", ".*"+GenericTestOperator.class.getSimpleName());
- props.put(StreamingApplication.DT_PREFIX + "template.t2.myStringProperty", "myStringPropertyValue");
-
- // direct setting
- props.put(StreamingApplication.DT_PREFIX + "operator.operator3.emitFormat", "emitFormatValue");
-
- LogicalPlan dag = new LogicalPlan();
- Operator operator1 = dag.addOperator("operator1", new ValidationTestOperator());
- Operator operator2 = dag.addOperator("operator2", new ValidationTestOperator());
- Operator operator3 = dag.addOperator("operator3", new GenericTestOperator());
-
- LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false));
- LOG.debug("calling addFromProperties");
- pb.addFromProperties(props, null);
-
- Map<String, String> configProps = pb.getProperties(dag.getMeta(operator1), "appName");
- Assert.assertEquals("" + configProps, 2, configProps.size());
- Assert.assertEquals("" + configProps, "stringProperty2Value-matchId1", configProps.get("stringProperty2"));
- Assert.assertEquals("" + configProps, "nested.propertyValue-matchId1", configProps.get("nested.property"));
-
- configProps = pb.getProperties(dag.getMeta(operator2), "appName");
- Assert.assertEquals("" + configProps, 1, configProps.size());
- Assert.assertEquals("" + configProps, "stringProperty2Value-matchClass1", configProps.get("stringProperty2"));
-
- configProps = pb.getProperties(dag.getMeta(operator3), "appName");
- Assert.assertEquals("" + configProps, 2, configProps.size());
- Assert.assertEquals("" + configProps, "myStringPropertyValue", configProps.get("myStringProperty"));
- Assert.assertEquals("" + configProps, "emitFormatValue", configProps.get("emitFormat"));
-
- }
-
- @Test
- public void testSetOperatorProperties() {
-
- Configuration conf = new Configuration(false);
- conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.myStringProperty", "myStringPropertyValue");
- conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.stringArrayField", "a,b,c");
- conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty.key1", "key1Val");
- conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key1.dot)", "key1dotVal");
- conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key2.dot)", "key2dotVal");
-
- LogicalPlan dag = new LogicalPlan();
- GenericTestOperator o1 = dag.addOperator("o1", new GenericTestOperator());
- ValidationTestOperator o2 = dag.addOperator("o2", new ValidationTestOperator());
-
- LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
-
- pb.setOperatorProperties(dag, "testSetOperatorProperties");
- Assert.assertEquals("o1.myStringProperty", "myStringPropertyValue", o1.getMyStringProperty());
- Assert.assertArrayEquals("o2.stringArrayField", new String[] {"a", "b", "c"}, o2.getStringArrayField());
-
- Assert.assertEquals("o2.mapProperty.key1", "key1Val", o2.getMapProperty().get("key1"));
- Assert.assertEquals("o2.mapProperty(key1.dot)", "key1dotVal", o2.getMapProperty().get("key1.dot"));
- Assert.assertEquals("o2.mapProperty(key2.dot)", "key2dotVal", o2.getMapProperty().get("key2.dot"));
-
- }
-
- @ApplicationAnnotation(name="AnnotatedAlias")
- class AnnotatedApplication implements StreamingApplication {
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- }
-
- }
-
- @Test
- public void testAppNameAttribute() {
- StreamingApplication app = new AnnotatedApplication();
- Configuration conf = new Configuration(false);
- conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
-
- LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
-
- Properties properties = new Properties();
- properties.put(StreamingApplication.DT_PREFIX + "application.TestAliasApp.class", app.getClass().getName());
-
- builder.addFromProperties(properties, null);
-
- LogicalPlan dag = new LogicalPlan();
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
- dag.setAttribute(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME, "testApp");
- builder.prepareDAG(dag, app, appPath);
-
- Assert.assertEquals("Application name", "testApp", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
- }
-
- @Test
- public void testAppAlias() {
- StreamingApplication app = new AnnotatedApplication();
- Configuration conf = new Configuration(false);
- conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
-
- LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
-
- Properties properties = new Properties();
- properties.put(StreamingApplication.DT_PREFIX + "application.TestAliasApp.class", app.getClass().getName());
-
- builder.addFromProperties(properties, null);
-
- LogicalPlan dag = new LogicalPlan();
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
- builder.prepareDAG(dag, app, appPath);
-
- Assert.assertEquals("Application name", "TestAliasApp", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
- }
-
-
- @Test
- public void testAppAnnotationAlias() {
- StreamingApplication app = new AnnotatedApplication();
- Configuration conf = new Configuration(false);
- conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
-
- LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
-
- LogicalPlan dag = new LogicalPlan();
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
- builder.prepareDAG(dag, app, appPath);
-
- Assert.assertEquals("Application name", "AnnotatedAlias", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
- }
-
- @Test
- @SuppressWarnings( {"UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes"})
- public void testOperatorLevelAttributes() {
- String appName = "app1";
- StreamingApplication app = new StreamingApplication() {
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- dag.addOperator("operator1", GenericTestOperator.class);
- dag.addOperator("operator2", GenericTestOperator.class);
- }
- };
-
- Properties props = new Properties();
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
- props.put(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "2");
- props.put(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.STATS_LISTENERS.getName(), PartitionLoadWatch.class.getName());
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "20");
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- Assert.assertEquals("", Integer.valueOf(20), dag.getOperatorMeta("operator1").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
- Assert.assertEquals("", Integer.valueOf(2), dag.getOperatorMeta("operator2").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
- Assert.assertEquals("", PartitionLoadWatch.class, dag.getOperatorMeta("operator2").getValue(OperatorContext.STATS_LISTENERS).toArray()[0].getClass());
- }
-
- @Test
- public void testOperatorLevelProperties() {
- String appName = "app1";
- final GenericTestOperator operator1 = new GenericTestOperator();
- final GenericTestOperator operator2 = new GenericTestOperator();
- StreamingApplication app = new StreamingApplication() {
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- dag.addOperator("operator1", operator1);
- dag.addOperator("operator2", operator2);
- }
- };
-
- Properties props = new Properties();
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
- props.put(StreamingApplication.DT_PREFIX + "operator.*.myStringProperty", "pv1");
- props.put(StreamingApplication.DT_PREFIX + "operator.*.booleanProperty", Boolean.TRUE.toString());
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1.myStringProperty", "apv1");
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- Assert.assertEquals("apv1", operator1.getMyStringProperty());
- Assert.assertEquals("pv1", operator2.getMyStringProperty());
- Assert.assertEquals(true, operator2.isBooleanProperty());
- }
-
- @Test
- public void testApplicationLevelParameter()
- {
- String appName = "app1";
- final GenericTestOperator operator1 = new GenericTestOperator();
- final GenericTestOperator operator2 = new GenericTestOperator();
- StreamingApplication app = new StreamingApplication()
- {
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- dag.addOperator("operator1", operator1);
- dag.addOperator("operator2", operator2);
- }
- };
-
- Properties props = new Properties();
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
- props.put(StreamingApplication.DT_PREFIX + "operator.*.myStringProperty", "foo ${xyz} bar ${zzz} baz");
- props.put(StreamingApplication.DT_PREFIX + "operator.*.booleanProperty", Boolean.TRUE.toString());
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1.myStringProperty", "apv1");
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
-
- Configuration vars = new Configuration(false);
- vars.set("xyz", "123");
- vars.set("zzz", "456");
- dagBuilder.addFromProperties(props, vars);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- Assert.assertEquals("apv1", operator1.getMyStringProperty());
- Assert.assertEquals("foo 123 bar 456 baz", operator2.getMyStringProperty());
- Assert.assertEquals(true, operator2.isBooleanProperty());
- }
-
- @Test
- @SuppressWarnings("UnnecessaryBoxing")
- public void testPortLevelAttributes() {
- String appName = "app1";
- SimpleTestApplication app = new SimpleTestApplication();
-
- Properties props = new Properties();
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1.port.*." + PortContext.QUEUE_CAPACITY.getName(), "" + 16 * 1024);
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator2.inputport.inport1." + PortContext.QUEUE_CAPACITY.getName(), "" + 32 * 1024);
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator2.outputport.outport1." + PortContext.QUEUE_CAPACITY.getName(), "" + 32 * 1024);
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator3.port.*." + PortContext.QUEUE_CAPACITY.getName(), "" + 16 * 1024);
- props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator3.inputport.inport2." + PortContext.QUEUE_CAPACITY.getName(), "" + 32 * 1024);
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- OperatorMeta om1 = dag.getOperatorMeta("operator1");
- Assert.assertEquals("", Integer.valueOf(16 * 1024), om1.getMeta(app.gt1.outport1).getValue(PortContext.QUEUE_CAPACITY));
- OperatorMeta om2 = dag.getOperatorMeta("operator2");
- Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(app.gt2.inport1).getValue(PortContext.QUEUE_CAPACITY));
- Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(app.gt2.outport1).getValue(PortContext.QUEUE_CAPACITY));
- OperatorMeta om3 = dag.getOperatorMeta("operator3");
- Assert.assertEquals("", Integer.valueOf(16 * 1024), om3.getMeta(app.gt3.inport1).getValue(PortContext.QUEUE_CAPACITY));
- Assert.assertEquals("", Integer.valueOf(32 * 1024), om3.getMeta(app.gt3.inport2).getValue(PortContext.QUEUE_CAPACITY));
- }
-
-
- @Test
- public void testInvalidAttribute() throws Exception {
- Assert.assertNotSame(0, com.datatorrent.api.Context.DAGContext.serialVersionUID);
- Attribute<String> attribute = new Attribute<>("", null);
-
- Field nameField = Attribute.class.getDeclaredField("name");
- nameField.setAccessible(true);
- nameField.set(attribute, "NOT_CONFIGURABLE");
- nameField.setAccessible(false);
-
- ContextUtils.addAttribute(com.datatorrent.api.Context.DAGContext.class, attribute);
- AttributeParseUtils.initialize();
- ConfElement.initialize();
-
- // attribute that cannot be configured
-
- Properties props = new Properties();
- props.put(StreamingApplication.DT_PREFIX + "attr.NOT_CONFIGURABLE", "value");
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- try {
- dagBuilder.prepareDAG(new LogicalPlan(), null, "");
- Assert.fail("Exception expected");
- } catch (Exception e) {
- Assert.assertThat("Attribute not configurable", e.getMessage(), RegexMatcher.matches("Attribute does not support property configuration: NOT_CONFIGURABLE.*"));
- }
-
- ContextUtils.removeAttribute(com.datatorrent.api.Context.DAGContext.class, attribute);
- AttributeParseUtils.initialize();
- ConfElement.initialize();
-
- // invalid attribute name
- props = new Properties();
- String invalidAttribute = StreamingApplication.DT_PREFIX + "attr.INVALID_NAME";
- props.put(invalidAttribute, "value");
-
- try {
- new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
- Assert.fail("Exception expected");
- } catch (Exception e) {
- LOG.debug("Exception message: {}", e.getMessage());
- Assert.assertThat("Invalid attribute name", e.getMessage(), RegexMatcher.matches("Invalid attribute reference: " + invalidAttribute));
- }
- }
-
- @Test
- public void testAttributesCodec() {
- Assert.assertNotSame(null, new Long[] {com.datatorrent.api.Context.DAGContext.serialVersionUID, OperatorContext.serialVersionUID, PortContext.serialVersionUID});
- @SuppressWarnings("unchecked")
- Set<Class<? extends Context>> contextClasses = Sets.newHashSet(com.datatorrent.api.Context.DAGContext.class, OperatorContext.class, PortContext.class);
- for (Class<?> c : contextClasses) {
- for (Attribute<Object> attr : AttributeInitializer.getAttributes(c)) {
- Assert.assertNotNull(attr.name + " codec", attr.codec);
- }
- }
- }
-
- @Test
- public void testTupleClassAttr() throws Exception
- {
- String resourcePath = "/schemaTestTopology.json";
- InputStream is = this.getClass().getResourceAsStream(resourcePath);
- if (is == null) {
- fail("Could not load " + resourcePath);
- }
- StringWriter writer = new StringWriter();
-
- IOUtils.copy(is, writer);
- JSONObject json = new JSONObject(writer.toString());
-
- Configuration conf = new Configuration(false);
-
- LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
- LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
- dag.validate();
-
- OperatorMeta operator1 = dag.getOperatorMeta("operator1");
- assertEquals("operator1.classname", SchemaTestOperator.class, operator1.getOperator().getClass());
-
- StreamMeta input1 = dag.getStream("inputStream");
- assertNotNull(input1);
- for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
- Assert.assertEquals("tuple class name required", TestSchema.class, targetPort.getAttributes().get(PortContext.TUPLE_CLASS));
- }
- }
-
- @Test(expected = ValidationException.class)
- public void testTupleClassAttrValidation() throws Exception
- {
- String resourcePath = "/schemaTestTopology.json";
- InputStream is = this.getClass().getResourceAsStream(resourcePath);
- if (is == null) {
- fail("Could not load " + resourcePath);
- }
- StringWriter writer = new StringWriter();
-
- IOUtils.copy(is, writer);
- JSONObject json = new JSONObject(writer.toString());
-
- //removing schema so that validation fails
- json.getJSONArray("streams").getJSONObject(0).remove("schema");
- Configuration conf = new Configuration(false);
-
- LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
- LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
-
- dag.validate();
- }
-
- @Test
- public void testTestTupleClassAttrSetFromConfig()
- {
- Configuration conf = new Configuration(false);
- conf.set(StreamingApplication.DT_PREFIX + "operator.o2.port.schemaRequiredPort.attr.TUPLE_CLASS",
- "com.datatorrent.stram.plan.LogicalPlanConfigurationTest$TestSchema");
-
- StreamingApplication streamingApplication = new StreamingApplication()
- {
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- TestGeneratorInputOperator o1 = dag.addOperator("o1", new TestGeneratorInputOperator());
- SchemaTestOperator o2 = dag.addOperator("o2", new SchemaTestOperator());
- dag.addStream("stream", o1.outport, o2.schemaRequiredPort);
- }
- };
- LogicalPlan dag = new LogicalPlan();
- LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
- lpc.prepareDAG(dag, streamingApplication, "app");
- dag.validate();
- }
-
- /**
- * This test and all of the following ambiguous attribute tests verify that when an ambiguous attribute
- * name is provided, all the corresponding attributes are set.
- * <br/><br/>
- * <b>Note:</b> Ambiguous attribute means that when multiple attributes with the same
- * simple name exist for multiple types of dag elements (like operators and ports).
- * An example of such attributes are the com.datatorrent.api.Context.OperatorContext.AUTO_RECORD
- * and com.datatorrent.api.Context.PortContext.AUTO_RECORD.
- * <br/><br/>
- * This test should set the attribute on the operators and ports.
- */
- /**
- * This test should set the attribute on the operators and ports.
- */
- @Test
- public void testRootLevelAmbiguousAttributeSimple()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX,
- null,
- Boolean.TRUE,
- true,
- true);
- }
-
- /**
- * This test should set the attribute on the operators and ports.
- */
- @Test
- public void testApplicationLevelAmbiguousAttributeSimple()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- null,
- Boolean.TRUE,
- true,
- true);
- }
-
- /**
- * This should only set the attribute on the operator
- */
- @Test
- public void testOperatorLevelAmbiguousAttributeSimple()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- null,
- Boolean.TRUE,
- true,
- false);
- }
-
- /**
- * This should only set the attribute on the port
- */
- @Test
- public void testPortLevelAmbiguousAttributeSimple()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX
- + "port"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- null,
- Boolean.TRUE,
- false,
- true);
- }
-
- /**
- * This test should set the attribute on the operators and ports.
- */
- @Test
- public void testRootLevelAmbiguousAttributeComplex()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX,
- PortContext.class.getCanonicalName(),
- Boolean.TRUE,
- false,
- true);
- }
-
- /**
- * This test should set the attribute on the operators and ports.
- */
- @Test
- public void testApplicationLevelAmbiguousAttributeComplex()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- PortContext.class.getCanonicalName(),
- Boolean.TRUE,
- false,
- true);
- }
-
- /**
- * This should only set the attribute on the operator
- */
- @Test
- public void testOperatorLevelAmbiguousAttributeComplex()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- OperatorContext.class.getCanonicalName(),
- Boolean.TRUE,
- true,
- false);
- }
-
- /**
- * This should only set the attribute on the port
- */
- @Test
- public void testOperatorLevelAmbiguousAttributeComplex2()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- PortContext.class.getCanonicalName(),
- Boolean.TRUE,
- false,
- true);
- }
-
- /**
- * This should only set the attribute on the port
- */
- @Test
- public void testPortLevelAmbiguousAttributeComplex()
- {
- testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD,
- Context.PortContext.AUTO_RECORD,
- StreamingApplication.DT_PREFIX
- + "port"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- PortContext.class.getCanonicalName(),
- Boolean.TRUE,
- false,
- true);
- }
-
- private void testAttributeAmbiguousSimpleHelper(Attribute<?> attributeObjOperator,
- Attribute<?> attributeObjPort,
- String root,
- String contextClass,
- Object val,
- boolean operatorSet,
- boolean portSet)
- {
- Properties props = propertiesBuilder(attributeObjOperator.getSimpleName(),
- root,
- contextClass,
- val);
-
- simpleAttributeOperatorHelperAssert(attributeObjOperator,
- props,
- val,
- operatorSet);
-
- simpleNamePortAssertHelperAssert(attributeObjPort,
- props,
- val,
- portSet);
- }
-
- @Test
- public void testRootLevelAttributeSimpleNameOperator()
- {
- simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB,
- StreamingApplication.DT_PREFIX,
- true,
- (Integer)4096,
- true,
- true);
- }
-
- @Test
- public void testRootLevelStorageAgentSimpleNameOperator()
- {
- MockStorageAgent mockAgent = new MockStorageAgent();
-
- simpleAttributeOperatorHelper(OperatorContext.STORAGE_AGENT,
- StreamingApplication.DT_PREFIX,
- true,
- mockAgent,
- true,
- false);
- }
-
- @Test
- public void testRootLevelAttributeSimpleNameOperatorNoScope()
- {
- simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB,
- StreamingApplication.DT_PREFIX,
- true,
- (Integer)4096,
- true,
- false);
- }
-
- @Test
- public void testApplicationLevelAttributeSimpleNameOperator()
- {
- simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "SimpleTestApp"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- true,
- (Integer)4096,
- true,
- true);
- }
-
- private void simpleAttributeOperatorHelper(Attribute<?> attributeObj,
- String root,
- boolean simpleName,
- Object val,
- boolean set,
- boolean scope)
- {
- Properties props = propertiesBuilderOperator(attributeObj.getSimpleName(),
- root,
- simpleName,
- val,
- scope);
-
- simpleAttributeOperatorHelperAssert(attributeObj,
- props,
- val,
- set);
- }
-
- private void simpleAttributeOperatorHelperAssert(Attribute<?> attributeObj,
- Properties props,
- Object val,
- boolean set)
- {
- SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- OperatorMeta om1 = dag.getOperatorMeta("operator1");
-
- if (set) {
- Assert.assertEquals(val, om1.getValue(attributeObj));
- } else {
- Assert.assertNotEquals(val, om1.getValue(attributeObj));
- }
-
- OperatorMeta om2 = dag.getOperatorMeta("operator2");
-
- if (set) {
- Assert.assertEquals(val, om2.getValue(attributeObj));
- } else {
- Assert.assertNotEquals(val, om2.getValue(attributeObj));
- }
-
- OperatorMeta om3 = dag.getOperatorMeta("operator3");
-
- if (set) {
- Assert.assertEquals(val, om3.getValue(attributeObj));
- } else {
- Assert.assertNotEquals(val, om3.getValue(attributeObj));
- }
- }
-
- /* Port tests */
- @Test
- public void testRootLevelAttributeSimpleNamePort()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- true,
- (Integer)4096,
- true,
- true);
- }
-
- @Test
- public void testRootLevelAttributeSimpleNamePortNoScope()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- true,
- (Integer)4096,
- true,
- false);
- }
-
- @Test
- public void testOperatorLevelAttributeSimpleNamePort()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- true,
- (Integer)4096,
- true,
- true);
- }
-
- @Test
- public void testApplicationLevelAttributeSimpleNamePort()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "SimpleTestApp"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- true,
- (Integer)4096,
- true,
- true);
- }
-
- @Test
- public void testRootLevelAttributeComplexNamePort()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- false,
- (Integer)4096,
- true,
- true);
- }
-
- @Test
- public void testRootLevelAttributeComplexNamePortNoScope()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- false,
- (Integer)4096,
- true,
- false);
- }
-
- @Test
- public void testOperatorLevelAttributeComplexNamePort()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- false,
- (Integer)4096,
- true,
- true);
- }
-
- @Test
- public void testApplicationLevelAttributeComplexNamePort()
- {
- simpleAttributePortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "SimpleTestApp"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- false,
- (Integer)4096,
- true,
- true);
- }
-
- /* Input port tests */
- @Test
- public void testRootLevelAttributeSimpleNameInputPort()
- {
- simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- true,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testOperatorLevelAttributeSimpleNameInputPort()
- {
- simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- true,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testApplicationLevelAttributeSimpleNameInputPort()
- {
- simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "SimpleTestApp"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- true,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testRootLevelAttributeComplexNameInputPort()
- {
- simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- false,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testOperatorLevelAttributeComplexNameInputPort()
- {
- simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- false,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testApplicationLevelAttributeComplexNameInputPort()
- {
- simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "SimpleTestApp"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- false,
- (Integer)4096,
- true);
- }
-
- /* Output port tests */
- @Test
- public void testRootLevelAttributeSimpleNameOutputPort()
- {
- simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- true,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testOperatorLevelAttributeSimpleNameOutputPort()
- {
- simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- true,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testApplicationLevelAttributeSimpleNameOutputPort()
- {
- simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "SimpleTestApp"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- true,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testRootLevelAttributeComplexNameOutputPort()
- {
- simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX,
- false,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testOperatorLevelAttributeComplexNameOutputPort()
- {
- simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- false,
- (Integer)4096,
- true);
- }
-
- @Test
- public void testApplicationLevelAttributeComplexNameOutputPort()
- {
- simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY,
- StreamingApplication.DT_PREFIX
- + "application"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "SimpleTestApp"
- + LogicalPlanConfiguration.KEY_SEPARATOR,
- false,
- (Integer)4096,
- true);
- }
-
- /* Helpers for building ports */
- private void simpleAttributePortHelper(Attribute<?> attributeObj,
- String root,
- boolean simpleName,
- Object val,
- boolean set,
- boolean scope)
- {
- Properties props = propertiesBuilderPort(attributeObj.getSimpleName(),
- root,
- simpleName,
- val,
- scope);
-
- simpleNamePortAssertHelperAssert(attributeObj,
- props,
- val,
- set);
- }
-
- private void simpleAttributeInputPortHelper(Attribute<?> attributeObj,
- String root,
- boolean simpleName,
- Object val,
- boolean set)
- {
- Properties props = propertiesBuilderInputPort(attributeObj.getSimpleName(),
- root,
- simpleName,
- val);
-
- simpleNameInputPortAssertHelperAssert(attributeObj,
- props,
- val,
- set);
-
- simpleNameOutputPortAssertHelperAssert(attributeObj,
- props,
- val,
- !set);
- }
-
- private void simpleAttributeOutputPortHelper(Attribute<?> attributeObj,
- String root,
- boolean simpleName,
- Object val,
- boolean set)
- {
- Properties props = propertiesBuilderOutputPort(attributeObj.getSimpleName(),
- root,
- simpleName,
- val);
-
- simpleNameOutputPortAssertHelperAssert(attributeObj,
- props,
- val,
- set);
-
- simpleNameInputPortAssertHelperAssert(attributeObj,
- props,
- val,
- !set);
- }
-
- private void simpleNamePortAssertHelperAssert(Attribute<?> attributeObj,
- Properties props,
- Object val,
- boolean set)
- {
- SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- simpleNamePortAssertHelper(attributeObj,
- dag,
- "operator1",
- val,
- set);
-
- simpleNamePortAssertHelper(attributeObj,
- dag,
- "operator2",
- val,
- set);
-
- simpleNamePortAssertHelper(attributeObj,
- dag,
- "operator3",
- val,
- set);
- }
-
- private void simpleNameInputPortAssertHelperAssert(Attribute<?> attributeObj,
- Properties props,
- Object val,
- boolean set)
- {
- SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- simpleNameInputPortAssertHelper(attributeObj,
- dag,
- "operator1",
- val,
- set);
-
- simpleNameInputPortAssertHelper(attributeObj,
- dag,
- "operator2",
- val,
- set);
-
- simpleNameInputPortAssertHelper(attributeObj,
- dag,
- "operator3",
- val,
- set);
- }
-
- private void simpleNameOutputPortAssertHelperAssert(Attribute<?> attributeObj,
- Properties props,
- Object val,
- boolean set)
- {
- SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
-
- LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
- dagBuilder.addFromProperties(props, null);
-
- String appPath = app.getClass().getName().replace(".", "/") + ".class";
-
- LogicalPlan dag = new LogicalPlan();
- dagBuilder.prepareDAG(dag, app, appPath);
-
- simpleNameOutputPortAssertHelper(attributeObj,
- dag,
- "operator1",
- val,
- set);
-
- simpleNameOutputPortAssertHelper(attributeObj,
- dag,
- "operator2",
- val,
- set);
-
- simpleNameOutputPortAssertHelper(attributeObj,
- dag,
- "operator3",
- val,
- set);
- }
-
- private void simpleNamePortAssertHelper(Attribute<?> attributeObj,
- LogicalPlan dag,
- String operatorName,
- Object queueCapacity,
- boolean set)
- {
- simpleNameInputPortAssertHelper(attributeObj,
- dag,
- operatorName,
- queueCapacity,
- set);
-
- simpleNameOutputPortAssertHelper(attributeObj,
- dag,
- operatorName,
- queueCapacity,
- set);
- }
-
- private void simpleNameInputPortAssertHelper(Attribute<?> attributeObj,
- LogicalPlan dag,
- String operatorName,
- Object queueCapacity,
- boolean set)
- {
- OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
-
- for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
- if (set) {
- Assert.assertEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
- } else {
- Assert.assertNotEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
- }
- }
- }
-
- private void simpleNameOutputPortAssertHelper(Attribute<?> attributeObj,
- LogicalPlan dag,
- String operatorName,
- Object queueCapacity,
- boolean set)
- {
- OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
-
- for (OutputPortMeta outputPortMeta: operatorMeta.getOutputStreams().keySet()) {
- if (set) {
- Assert.assertEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
- } else {
- Assert.assertNotEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
- }
- }
- }
-
- /* Helpers for building properties */
- private Properties propertiesBuilder(String attributeName,
- String root,
- String contextClass,
- Object val)
- {
- boolean simpleName = contextClass == null;
-
- if (!simpleName) {
- attributeName = contextClass
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + attributeName;
- }
-
- Properties props = new Properties();
-
- String propName = root
- + StramElement.ATTR.getValue()
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + attributeName;
-
- LOG.debug("adding prop {} with value {}", propName, val.toString());
-
- props.put(propName,
- val.toString());
-
- return props;
- }
-
- private Properties propertiesBuilderOperator(String attributeName,
- String root,
- boolean simpleName,
- Object val,
- boolean addOperator)
- {
- String contextClass = simpleName ? null : OperatorContext.class.getCanonicalName();
-
- if (addOperator) {
- root += "operator"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR;
- }
-
- return propertiesBuilder(attributeName,
- root,
- contextClass,
- val);
- }
-
- private Properties propertiesBuilderPort(String attributeName,
- String root,
- boolean simpleName,
- Object val,
- boolean addPort)
- {
- String contextClass = simpleName ? null : PortContext.class.getCanonicalName();
-
- if (addPort) {
- root += "port"
- + LogicalPlanConfiguration.KEY_SEPARATOR
- + "*"
- + LogicalPlanConfiguration.KEY_SEPARATOR;
- }
-
- return propertiesBuilder(attributeName,
- root,
- contextClass,
- val);
- }
-
- private Properties propertiesBuilderInputPort(String attributeName,
- String root,
- boolean simpleName,
- Object val)
- {
- String contextClass = simpleName ? null: PortContext.class.getCanonicalName();
-
- root += "inputport" +
- LogicalPlanConfiguration.KEY_SEPARATOR +
- "*" +
- LogicalPlanConfiguration.KEY_SEPARATOR;
-
- return propertiesBuilder(attributeName,
- root,
- contextClass,
- val);
- }
-
- private Properties propertiesBuilderOutputPort(String attributeName,
- String root,
- boolean simpleName,
- Object val)
- {
- String contextClass = simpleName ? null: PortContext.class.getCanonicalName();
-
- root += "outputport" +
- LogicalPlanConfiguration.KEY_SEPARATOR +
- "*" +
- LogicalPlanConfiguration.KEY_SEPARATOR;
-
- return propertiesBuilder(attributeName,
- root,
- contextClass,
- val);
- }
-
- private static final Logger logger = LoggerFactory.getLogger(LogicalPlanConfigurationTest.class);
-
- public static class TestApplication implements StreamingApplication {
- Integer testprop1;
- Integer testprop2;
- Integer testprop3;
- TestInnerClass inncls;
- public TestApplication() {
- inncls=new TestInnerClass();
- }
-
- public Integer getTestprop1() {
- return testprop1;
- }
-
- public void setTestprop1(Integer testprop1) {
- this.testprop1 = testprop1;
- }
-
- public Integer getTestprop2() {
- return testprop2;
- }
-
- public void setTestprop2(Integer testprop2) {
- this.testprop2 = testprop2;
- }
-
- public Integer getTestprop3() {
- return testprop3;
- }
-
- public void setTestprop3(Integer testprop3) {
- this.testprop3 = testprop3;
- }
-
- public TestInnerClass getInncls() {
- return inncls;
- }
-
- public void setInncls(TestInnerClass inncls) {
- this.inncls = inncls;
- }
-
- @Override
- public void populateDAG(DAG dag, Configuration conf) {
-
- }
- public class TestInnerClass{
- Integer a;
-
- public Integer getA() {
- return a;
- }
-
- public void setA(Integer a) {
- this.a = a;
- }
- }
- }
-
- public static class TestStatsListener implements StatsListener{
-
- private int intProp;
-
- public TestStatsListener()
- {
- }
-
- @Override
- public Response processStats(BatchedOperatorStats stats)
- {
- return null;
- }
-
- public int getIntProp()
- {
- return intProp;
- }
-
- public void setIntProp(int intProp)
- {
- this.intProp = intProp;
- }
-
- @Override
- public int hashCode()
- {
- final int prime = 31;
- int result = 1;
- result = prime * result + intProp;
- return result;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TestStatsListener other = (TestStatsListener) obj;
- if (intProp != other.intProp)
- return false;
- return true;
- }
- }
-
- public static class TestSchema
- {
- }
-
- public static class SimpleTestApplication implements StreamingApplication
- {
- public final GenericTestOperator gt1 = new GenericTestOperator();
- public final GenericTestOperator gt2 = new GenericTestOperator();
- public final GenericTestOperator gt3 = new GenericTestOperator();
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- dag.addOperator("operator1", gt1);
- dag.addOperator("operator2", gt2);
- dag.addOperator("operator3", gt3);
- dag.addStream("s1", gt1.outport1, gt2.inport1);
- dag.addStream("s2", gt2.outport1, gt3.inport1, gt3.inport2);
- }
- };
-
- @ApplicationAnnotation(name="SimpleTestApp")
- public static class SimpleTestApplicationWithName extends SimpleTestApplication
- {
- };
-
- private static final Logger LOG = LoggerFactory.getLogger(LogicalPlanConfigurationTest.class);
-}
-
[7/8] incubator-apex-core git commit: APEX-28 #resolve
Posted by cs...@apache.org.
APEX-28 #resolve
- Rename of files requires a separate commit to preserve attribution.
- Improved documentation
- Added unit test to make sure that attributes declared in multiple contexts have the same type.
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/977093e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/977093e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/977093e1
Branch: refs/heads/devel-3
Commit: 977093e171f1183985ae80d42b0d6dbc3af6cbc5
Parents: 434a717
Author: Timothy Farkas <ti...@datatorrent.com>
Authored: Tue Aug 25 18:03:08 2015 -0700
Committer: Timothy Farkas <ti...@datatorrent.com>
Committed: Wed Sep 16 15:31:44 2015 -0700
----------------------------------------------------------------------
.../java/com/datatorrent/api/Attribute.java | 11 +-
.../main/java/com/datatorrent/api/Context.java | 10 -
.../stram/plan/logical/LogicalPlan.java | 6 +-
.../plan/logical/LogicalPlanConfiguration.java | 472 +++--
.../plan/LogicalPlanConfigurationTest.java | 1788 ------------------
.../datatorrent/stram/plan/LogicalPlanTest.java | 990 ----------
.../logical/LogicalPlanConfigurationTest.java | 1511 +++++++++++++++
.../stram/plan/logical/LogicalPlanTest.java | 988 ++++++++++
.../src/test/resources/schemaTestTopology.json | 2 +-
9 files changed, 2785 insertions(+), 2993 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/api/src/main/java/com/datatorrent/api/Attribute.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Attribute.java b/api/src/main/java/com/datatorrent/api/Attribute.java
index 4c16a2a..a7492b5 100644
--- a/api/src/main/java/com/datatorrent/api/Attribute.java
+++ b/api/src/main/java/com/datatorrent/api/Attribute.java
@@ -277,13 +277,6 @@ public class Attribute<T> implements Serializable
if (map.containsKey(clazz)) {
return 0;
}
-
- map.put(clazz, getAttributesNoSave(clazz));
- return (long)clazz.getModifiers() << 32 | clazz.hashCode();
- }
-
- public static Set<Attribute<Object>> getAttributesNoSave(Class<?> clazz)
- {
Set<Attribute<Object>> set = new HashSet<Attribute<Object>>();
try {
for (Field f: clazz.getDeclaredFields()) {
@@ -330,8 +323,8 @@ public class Attribute<T> implements Serializable
catch (Exception ex) {
DTThrowable.rethrow(ex);
}
-
- return set;
+ map.put(clazz, set);
+ return (long)clazz.getModifiers() << 32 | clazz.hashCode();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/api/src/main/java/com/datatorrent/api/Context.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Context.java b/api/src/main/java/com/datatorrent/api/Context.java
index c2d974a..cd10398 100644
--- a/api/src/main/java/com/datatorrent/api/Context.java
+++ b/api/src/main/java/com/datatorrent/api/Context.java
@@ -33,16 +33,6 @@ import com.datatorrent.api.annotation.Stateless;
*/
public interface Context
{
- /*
- * Note: If the same name is given to an Attribute specified in multiple Context classes, then the type of that
- * Attribute is required to be the same accross all Context classes. This is required because if a simple attribute
- * name is specified in a properties file at the top level context then that attribute needs to be set in all child configurations. If
- * there were multiple Attributes specified in different Contexts with the same name, but a different type, then
- * it would not be possible to set the values of Attributes specified by a simple attribute name in the root
- * context of a properties file. If this were the case, then adding another Attribute with the same name as a pre-existing Attribute to a new Context
- * class would be a backwards incompatible change.
- */
-
/**
* Get the attributes associated with this context.
* The returned map does not contain any attributes that may have been defined in the parent context of this context.
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 8826896..94d18ba 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -1088,7 +1088,7 @@ public class LogicalPlan implements Serializable, DAG
if (e.getKey().getOperatorWrapper() == om) {
stream.sinks.remove(e.getKey());
}
- // If persistStream was enabled for stream, reset stream when sink removed
+ // If persistStream was enabled for stream, reset stream when sink removed
stream.resetStreamPersistanceOnSinkRemoval(e.getKey());
}
this.operators.remove(om.getName());
@@ -1431,11 +1431,11 @@ public class LogicalPlan implements Serializable, DAG
for (StreamMeta s: streams.values()) {
if (s.source == null) {
- throw new ValidationException(String.format("stream source not connected: %s", s.getName()));
+ throw new ValidationException("Stream source not connected: " + s.getName());
}
if (s.sinks.isEmpty()) {
- throw new ValidationException(String.format("stream sink not connected: %s", s.getName()));
+ throw new ValidationException("Stream sink not connected: " + s.getName());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index a3a18c2..7a53cd7 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -15,6 +15,7 @@
*/
package com.datatorrent.stram.plan.logical;
+
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -22,14 +23,17 @@ import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+
import java.util.*;
import java.util.Map.Entry;
-import jline.internal.Preconditions;
import javax.validation.ValidationException;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -61,7 +65,6 @@ import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
-import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.StramElement;
import com.datatorrent.stram.util.ObjectMapperFactory;
/**
@@ -159,43 +162,16 @@ public class LogicalPlanConfiguration {
*/
protected enum ConfElement
{
- @SuppressWarnings("SetReplaceableByEnumSet")
- STRAM(null,
- null,
- new HashSet<StramElement>(),
- null),
- @SuppressWarnings("SetReplaceableByEnumSet")
- APPLICATION(StramElement.APPLICATION,
- STRAM,
- new HashSet<StramElement>(),
- DAGContext.class),
- @SuppressWarnings("SetReplaceableByEnumSet")
- TEMPLATE(StramElement.TEMPLATE,
- STRAM,
- new HashSet<StramElement>(),
- null),
- @SuppressWarnings("SetReplaceableByEnumSet")
- GATEWAY(StramElement.GATEWAY,
- ConfElement.APPLICATION,
- new HashSet<StramElement>(),
- null),
- @SuppressWarnings("SetReplaceableByEnumSet")
- OPERATOR(StramElement.OPERATOR,
- ConfElement.APPLICATION,
- new HashSet<StramElement>(),
- OperatorContext.class),
- @SuppressWarnings("SetReplaceableByEnumSet")
- STREAM(StramElement.STREAM,
- ConfElement.APPLICATION,
- new HashSet<StramElement>(),
- null),
- PORT(StramElement.PORT,
- ConfElement.OPERATOR,
- Sets.newHashSet(StramElement.INPUT_PORT, StramElement.OUTPUT_PORT),
- PortContext.class);
-
- public static final Map<StramElement, ConfElement> STRAM_ELEMENT_TO_CONF_ELEMENT = Maps.newHashMap();
- public static final Map<Class<? extends Context>, ConfElement> CONTEXT_TO_CONF_ELEMENT = Maps.newHashMap();
+ STRAM(null, null, null, null),
+ APPLICATION(StramElement.APPLICATION, STRAM, null, DAGContext.class),
+ TEMPLATE(StramElement.TEMPLATE, STRAM, null, null),
+ GATEWAY(StramElement.GATEWAY, ConfElement.APPLICATION, null, null),
+ OPERATOR(StramElement.OPERATOR, ConfElement.APPLICATION, null, OperatorContext.class),
+ STREAM(StramElement.STREAM, ConfElement.APPLICATION, null, null),
+ PORT(StramElement.PORT, ConfElement.OPERATOR, EnumSet.of(StramElement.INPUT_PORT, StramElement.OUTPUT_PORT), PortContext.class);
+
+ protected static final Map<StramElement, ConfElement> STRAM_ELEMENT_TO_CONF_ELEMENT = Maps.newHashMap();
+ protected static final Map<Class<? extends Context>, ConfElement> CONTEXT_TO_CONF_ELEMENT = Maps.newHashMap();
static {
initialize();
@@ -246,12 +222,8 @@ public class LogicalPlanConfiguration {
}
if (!ContextUtils.CONTEXT_CLASSES.equals(confElementContextClasses)) {
- throw new IllegalStateException("All the context classes "
- + ContextUtils.CONTEXT_CLASSES
- + " found in "
- + Context.class
- + " are not used by ConfElements "
- + confElementContextClasses);
+ throw new IllegalStateException("All the context classes " + ContextUtils.CONTEXT_CLASSES + " found in "
+ + Context.class + " are not used by ConfElements " + confElementContextClasses);
}
}
@@ -312,16 +284,15 @@ public class LogicalPlanConfiguration {
this.element = element;
this.parent = parent;
- this.allRelatedElements.addAll(additionalRelatedElements);
+ if (additionalRelatedElements != null) {
+ this.allRelatedElements.addAll(additionalRelatedElements);
+ }
+
this.allRelatedElements.add(element);
this.contextClass = contextClass;
- if (contextClass != null) {
- this.contextAttributes = ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass);
- } else {
- this.contextAttributes = Sets.newHashSet();
- }
+ this.contextAttributes = contextClass != null ? ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass) : new HashSet<String>();
}
private void setAllChildAttributes(Set<String> allChildAttributes)
@@ -445,8 +416,7 @@ public class LogicalPlanConfiguration {
*
* @param conf The current {@link Conf} type.
* @return A path from the current {@link Conf} type to a root {@link Conf} type, which includes the current and root
- * {
- * @lin Conf} types.
+ * {@link Conf} types.
*/
public static List<StramElement> getPathFromChildToRootInclusive(StramElement conf)
{
@@ -471,8 +441,7 @@ public class LogicalPlanConfiguration {
*
* @param conf The current {@link Conf} type.
* @return A path from the root {@link Conf} type to the current {@link Conf} type, which includes the current and root
- * {
- * @lin Conf} types.
+ * {@link Conf} types.
*/
public static List<StramElement> getPathFromRootToChildInclusive(StramElement conf)
{
@@ -487,11 +456,9 @@ public class LogicalPlanConfiguration {
* @param child The current {@link Conf} type.
* @param parent The parent {@link Conf} type.
* @return A path from the current {@link Conf} type to a parent {@link Conf} type, which includes the current and parent
- * {
- * @lin Conf} types.
+ * {@link Conf} types.
*/
- public static List<StramElement> getPathFromChildToParentInclusive(StramElement child,
- StramElement parent)
+ public static List<StramElement> getPathFromChildToParentInclusive(StramElement child, StramElement parent)
{
ConfElement confElement = STRAM_ELEMENT_TO_CONF_ELEMENT.get(child);
@@ -528,11 +495,9 @@ public class LogicalPlanConfiguration {
* @param child The current {@link Conf} type.
* @param parent The parent {@link Conf} type.
* @return A path from the parent {@link Conf} type to the current {@link Conf} type, which includes the current and parent
- * {
- * @lin Conf} types.
+ * {@link Conf} types.
*/
- public static List<StramElement> getPathFromParentToChildInclusive(StramElement child,
- StramElement parent)
+ public static List<StramElement> getPathFromParentToChildInclusive(StramElement child, StramElement parent)
{
List<StramElement> path = getPathFromChildToParentInclusive(child,
parent);
@@ -548,8 +513,7 @@ public class LogicalPlanConfiguration {
* @return The {@link ConfElement} that contains the given attribute, or null if no {@link ConfElement} contains
* the given attribute.
*/
- public static ConfElement findConfElementWithAttribute(ConfElement current,
- String simpleAttributeName)
+ public static ConfElement findConfElementWithAttribute(ConfElement current, String simpleAttributeName)
{
if (current.getContextAttributes().contains(simpleAttributeName)) {
return current;
@@ -573,9 +537,7 @@ public class LogicalPlanConfiguration {
List<StramElement> path = ConfElement.getPathFromParentToChildInclusive(childConfElement.getStramElement(),
parentConf.getConfElement().getStramElement());
- for (int pathIndex = 1;
- pathIndex < path.size();
- pathIndex++) {
+ for (int pathIndex = 1; pathIndex < path.size(); pathIndex++) {
LOG.debug("Adding conf");
StramElement pathElement = path.get(pathIndex);
//Add the configurations we need to hold this attribute
@@ -593,12 +555,19 @@ public class LogicalPlanConfiguration {
@SuppressWarnings("unchecked")
protected static class ContextUtils
{
- public static final Map<Class<? extends Context>, Set<String>> CONTEXT_CLASS_TO_ATTRIBUTES;
- public static final Set<Class<? extends Context>> CONTEXT_CLASSES;
- public static final Map<Class<? extends Context>, Map<String, Attribute<?>>> CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE;
+ private static final Map<String, Type> ATTRIBUTES_TO_TYPE = Maps.newHashMap();
+ public static final Map<Class<? extends Context>, Set<String>> CONTEXT_CLASS_TO_ATTRIBUTES = Maps.newHashMap();
+ public static final Set<Class<? extends Context>> CONTEXT_CLASSES = Sets.newHashSet();
+ public static final Map<Class<? extends Context>, Map<String, Attribute<?>>> CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE = Maps.newHashMap();
static {
- CONTEXT_CLASSES = Sets.newHashSet();
+ initialize();
+ }
+
+ @VisibleForTesting
+ protected static void initialize()
+ {
+ CONTEXT_CLASSES.clear();
for (Class<?> clazz: Context.class.getDeclaredClasses()) {
if (!Context.class.isAssignableFrom(clazz)) {
@@ -608,9 +577,17 @@ public class LogicalPlanConfiguration {
CONTEXT_CLASSES.add((Class<? extends Context>)clazz);
}
- CONTEXT_CLASS_TO_ATTRIBUTES = Maps.newHashMap();
+ buildAttributeMaps(CONTEXT_CLASSES);
+ }
- for (Class<? extends Context> contextClass: CONTEXT_CLASSES) {
+ @VisibleForTesting
+ protected static void buildAttributeMaps(Set<Class<? extends Context>> contextClasses)
+ {
+ CONTEXT_CLASS_TO_ATTRIBUTES.clear();
+ CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.clear();
+ ATTRIBUTES_TO_TYPE.clear();
+
+ for (Class<? extends Context> contextClass: contextClasses) {
Set<String> contextAttributes = Sets.newHashSet();
Field[] fields = contextClass.getDeclaredFields();
@@ -620,19 +597,29 @@ public class LogicalPlanConfiguration {
continue;
}
+ Type fieldType = ((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0];
contextAttributes.add(field.getName());
+
+ Type existingType = ATTRIBUTES_TO_TYPE.get(field.getName());
+
+ if (existingType != null && !existingType.equals(fieldType)) {
+ throw new ValidationException("The attribute " + field.getName() +
+ " is defined with two different types in two different context classes: " +
+ fieldType + " and " + existingType + "\n" +
+ "Attributes with the same name are required to have the same type accross all Context classes.");
+ }
+
+ ATTRIBUTES_TO_TYPE.put(field.getName(), fieldType);
}
CONTEXT_CLASS_TO_ATTRIBUTES.put(contextClass, contextAttributes);
}
- CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE = Maps.newHashMap();
-
- for (Class<? extends Context> contextClass: CONTEXT_CLASSES) {
+ for (Class<? extends Context> contextClass: contextClasses) {
Map<String, Attribute<?>> simpleAttributeNameToAttribute = Maps.newHashMap();
CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.put(contextClass, simpleAttributeNameToAttribute);
- Set<Attribute<Object>> attributes = AttributeInitializer.getAttributesNoSave(contextClass);
+ Set<Attribute<Object>> attributes = AttributeInitializer.getAttributes(contextClass);
LOG.debug("context class {} and attributes {}", contextClass, attributes);
@@ -644,6 +631,7 @@ public class LogicalPlanConfiguration {
private ContextUtils()
{
+ //Private construct to prevent instantiation of utility class
}
/**
@@ -735,6 +723,7 @@ public class LogicalPlanConfiguration {
private AttributeParseUtils()
{
+ //Private construct to prevent instantiation of utility class
}
/**
@@ -782,11 +771,7 @@ public class LogicalPlanConfiguration {
{
if (element != null && element != StramElement.ATTR) {
- throw new IllegalArgumentException("The given "
- + StramElement.class
- + " must either have a value of null or "
- + StramElement.ATTR
- + " but it had a value of " + element);
+ throw new IllegalArgumentException("The given " + StramElement.class + " must either have a value of null or " + StramElement.ATTR + " but it had a value of " + element);
}
String attributeName;
@@ -823,9 +808,7 @@ public class LogicalPlanConfiguration {
public static Class<? extends Context> getContainingContextClass(String attributeName)
{
if (isSimpleAttributeName(attributeName)) {
- throw new IllegalArgumentException("The given attribute name "
- + attributeName
- + " is simple.");
+ throw new IllegalArgumentException("The given attribute name " + attributeName + " is simple.");
}
LOG.debug("Attribute Name {}", attributeName);
@@ -847,9 +830,7 @@ public class LogicalPlanConfiguration {
if (Context.class.isAssignableFrom(clazz)) {
contextClass = (Class<? extends Context>)clazz;
} else {
- throw new IllegalArgumentException("The provided context class name "
- + contextClassName
- + " is not valid.");
+ throw new IllegalArgumentException("The provided context class name " + contextClassName + " is not valid.");
}
} catch (ClassNotFoundException ex) {
throw new IllegalArgumentException(ex);
@@ -858,9 +839,7 @@ public class LogicalPlanConfiguration {
String simpleAttributeName = getSimpleAttributeName(attributeName);
if (!ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass).contains(simpleAttributeName)) {
- throw new ValidationException(simpleAttributeName
- + " is not a valid attribute of "
- + contextClass);
+ throw new ValidationException(simpleAttributeName + " is not a valid attribute of " + contextClass);
}
return contextClass;
@@ -879,9 +858,7 @@ public class LogicalPlanConfiguration {
}
if (attributeName.endsWith(KEY_SEPARATOR)) {
- throw new IllegalArgumentException("The given attribute name ends with \""
- + KEY_SEPARATOR
- + "\" so a simple name cannot be extracted.");
+ throw new IllegalArgumentException("The given attribute name ends with \"" + KEY_SEPARATOR + "\" so a simple name cannot be extracted.");
}
return attributeName.substring(attributeName.lastIndexOf(KEY_SEPARATOR) + 1, attributeName.length());
@@ -961,6 +938,13 @@ public class LogicalPlanConfiguration {
return (T)parentConf;
}
+ /**
+ * Gets an ancestor {@link Conf} of this {@link Conf} of the given {@link StramElement} type.
+ * @param <T> The {@link Conf} Class of the ancestor conf
+ * @param ancestorElement The {@link StramElement} representing the type of the ancestor {@link Conf}.
+ * @return The ancestor {@link Conf} of the corresponding {@link StramElement} type, or null if no ancestor {@link Conf} with
+ * the given {@link StramElement} type exists.
+ */
@SuppressWarnings("unchecked")
public <T extends Conf> T getAncestorConf(StramElement ancestorElement) {
if (getConfElement().getStramElement() == ancestorElement) {
@@ -973,6 +957,16 @@ public class LogicalPlanConfiguration {
}
}
+ /**
+ * This method retrieves a child {@link Conf} of the given {@link StramElement} type with the given name. If
+ * a child {@link Conf} with the given name and {@link StramElement} type doesn't exist, then it is added.
+ * @param <T> The type of the child {@link Conf}.
+ * @param id The name of the child {@link Conf}.
+ * @param childType The {@link StramElement} representing the type of the child {@link Conf}.
+ * @param clazz The {@link java.lang.Class} of the child {@link Conf} to add if a {@link Conf} of the given id
+ * and {@link StramElement} type is not present.
+ * @return A child {@link Conf} of this {@link Conf} with the given id and {@link StramElement} type.
+ */
public <T extends Conf> T getOrAddChild(String id, StramElement childType, Class<T> clazz) {
@SuppressWarnings("unchecked")
Map<String, T> elChildren = (Map<String, T>)children.get(childType);
@@ -999,6 +993,15 @@ public class LogicalPlanConfiguration {
properties.setDefaultProperties(defaults);
}
+ /**
+ * This method returns a list of all the child {@link Conf}s of this {@link Conf} with the matching name
+ * and {@link StramElement} type.
+ * @param <T> The types of the child {@link Conf}s.
+ * @param name The name of the child {@link Conf}s to return. If the name of the specified child {@link Conf}
+ * is null then configurations with the name specified as a {@link LogicalPlanConfiguration#WILDCARD} are matched.
+ * @param childType The {@link StramElement} corresponding to the type of a child {@link Conf}.
+ * @return The list of child {@link Conf}s with a matching name and {@link StramElement} type.
+ */
public <T extends Conf> List<T> getMatchingChildConf(String name, StramElement childType) {
List<T> childConfs = new ArrayList<>();
Map<String, T> elChildren = getChildren(childType);
@@ -1038,6 +1041,17 @@ public class LogicalPlanConfiguration {
return childConfs;
}
+ /**
+ * Returns the {@link Conf} corresponding to the given id from the given map. If a {@link Conf} with the
+ * given id is not present in the given map, then a new {@link Conf} of the given class is created and added
+ * to the map.
+ * @param <T> The type of the {@link Conf}s contained in the map.
+ * @param map The map to retrieve a {@link Conf} from or add a {@link Conf} to.
+ * @param id The name of the {@link Conf} to retrieve from or add to the given map.
+ * @param clazz The {@link java.lang.Class} of the {@link Conf} to add to the given map, if a {@link Conf} with
+ * the given name is not present in the given map.
+ * @return A {@link Conf} with the given name, contained in the given map.
+ */
protected <T extends Conf> T getOrAddConf(Map<String, T> map, String id, Class<T> clazz) {
T conf = map.get(id);
if (conf == null) {
@@ -1046,12 +1060,7 @@ public class LogicalPlanConfiguration {
conf = declaredConstructor.newInstance(new Object[] {});
conf.setId(id);
map.put(id, conf);
- } catch (IllegalAccessException |
- IllegalArgumentException |
- InstantiationException |
- NoSuchMethodException |
- SecurityException |
- InvocationTargetException e) {
+ } catch (IllegalAccessException | IllegalArgumentException | InstantiationException | NoSuchMethodException | SecurityException | InvocationTargetException e) {
LOG.error("Error instantiating configuration", e);
}
}
@@ -1470,6 +1479,19 @@ public class LogicalPlanConfiguration {
elementMaps.put(StramElement.OUTPUT_PORT, PortConf.class);
}
+ /**
+ * This is a helper method which performs the following checks:<br/><br/>
+ * <ol>
+ * <li>If the given {@link StramElement} corresponds to a {@link Conf} type which is
+ * the same as the type of the given {@link Conf}, then the given {@link Conf} is returned.</li>
+ * <li>If the given {@link StramElement} corresponds to a {@link Conf} type which is
+ * a valid parent {@link Conf} type for the given ancestorConf, then the given ancestor {@link Conf} is
+ * returned.</li>
+ * @param element The {@link StramElement} type corresponding to this {@link Conf} or
+ * to a valid ancestor {@link Conf}.
+ * @param ancestorConf The {@link Conf} to return.
+ * @return The given {@link Conf}, or null if the first call to this method passes a null {@link StramElement}.
+ */
private static Conf getConf(StramElement element, Conf ancestorConf) {
if (element == ancestorConf.getConfElement().getStramElement()) {
return ancestorConf;
@@ -1481,9 +1503,23 @@ public class LogicalPlanConfiguration {
}
StramElement parentElement = ConfElement.getAllowedParentConf(element);
Conf parentConf = getConf(parentElement, ancestorConf);
+
+ if(parentConf == null) {
+ throw new IllegalArgumentException("The given StramElement is not the same type as the given ancestorConf, " +
+ "and it is not a valid type for a parent conf.");
+ }
+
return parentConf.getOrAddChild(WILDCARD, element, elementMaps.get(element));
}
+ /**
+ * This method adds a child {@link Conf} with the given {@link StramElement} type and name to the given
+ * ancestorConf.
+ * @param element The {@link StramElement} of the child {@link Conf} to add to the given ancestorConf.
+ * @param name The name of the child {@link Conf} to add to the given ancestorConf.
+ * @param ancestorConf The {@link Conf} to add a child {@link Conf} to.
+ * @return The child {@link Conf} that was added to the given ancestorConf.
+ */
private static Conf addConf(StramElement element, String name, Conf ancestorConf) {
StramElement parentElement = ConfElement.getAllowedParentConf(element);
Conf conf1 = null;
@@ -1494,6 +1530,16 @@ public class LogicalPlanConfiguration {
return conf1;
}
+ /**
+ * This method returns a list of all the child {@link Conf}s of the given {@link List} of {@link Conf}s with the matching name
+ * and {@link StramElement} type.
+ * @param <T> The types of the child {@link Conf}s.
+ * @param confs The list of {@link Conf}s whose children will be searched.
+ * @param name The name of the child {@link Conf}s to return. If the name of the specified child {@link Conf}
+ * is null then configurations with the name specified as a {@link LogicalPlanConfiguration#WILDCARD} are matched.
+ * @param childType The {@link StramElement} corresponding to the type of a child {@link Conf}.
+ * @return The list of child {@link Conf}s with a matching name and {@link StramElement} type.
+ */
private <T extends Conf> List<T> getMatchingChildConf(List<? extends Conf> confs, String name, StramElement childType) {
List<T> childConfs = Lists.newArrayList();
for (Conf conf1 : confs) {
@@ -1685,7 +1731,7 @@ public class LogicalPlanConfiguration {
* @param index The current index that the parser is on for processing the property name.
* @param propertyName The original unsplit Apex property name.
* @param propertyValue The value corresponding to the Apex property.
- * @param conf
+ * @param conf The current {@link Conf} to add properties to.
*/
private void parseStramPropertyTokens(String[] keys, int index, String propertyName, String propertyValue, Conf conf) {
if (index < keys.length) {
@@ -1697,104 +1743,141 @@ public class LogicalPlanConfiguration {
if ((element == StramElement.APPLICATION) || (element == StramElement.OPERATOR) || (element == StramElement.STREAM)
|| (element == StramElement.PORT) || (element == StramElement.INPUT_PORT) || (element == StramElement.OUTPUT_PORT)
|| (element == StramElement.TEMPLATE)) {
- if ((index + 1) < keys.length) {
- String name = keys[index+1];
- Conf elConf = addConf(element, name, conf);
- if (elConf != null) {
- parseStramPropertyTokens(keys, index + 2, propertyName, propertyValue, elConf);
- } else {
- LOG.error("Invalid configuration key: {}", propertyName);
- }
- } else {
- LOG.warn("Invalid configuration key: {}", propertyName);
- }
- } else if ((element == StramElement.GATEWAY)) {
- Conf elConf = addConf(element, null, conf);
- if (elConf != null) {
- parseStramPropertyTokens(keys, index+1, propertyName, propertyValue, elConf);
- } else {
- LOG.error("Invalid configuration key: {}", propertyName);
- }
+ parseAppElement(index, keys, element, conf, propertyName, propertyValue);
+ } else if (element == StramElement.GATEWAY) {
+ parseGatewayElement(element, conf, keys, index, propertyName, propertyValue);
} else if ((element == StramElement.ATTR) || ((element == null) && (conf.getDefaultChildElement() == StramElement.ATTR))) {
- String attributeName = AttributeParseUtils.getAttributeName(element, keys, index);
-
- if (element != StramElement.ATTR) {
- String expName = getCompleteKey(keys, 0, index) + KEY_SEPARATOR + StramElement.ATTR.getValue() + KEY_SEPARATOR + attributeName;
- LOG.warn("Referencing the attribute as {} instead of {} is deprecated!", getCompleteKey(keys, 0), expName);
- }
-
- if (conf.getConfElement().getStramElement() == null) {
- conf = addConf(StramElement.APPLICATION, WILDCARD, conf);
- }
-
- if (conf != null) {
- if (AttributeParseUtils.isSimpleAttributeName(attributeName)) {
- //The provided attribute name was a simple name
-
- if (!AttributeParseUtils.ALL_SIMPLE_ATTRIBUTE_NAMES.contains(attributeName)) {
- throw new ValidationException("Invalid attribute reference: " + getCompleteKey(keys, 0));
- }
-
- if (!conf.getConfElement().getAllChildAttributes().contains(attributeName)) {
- throw new ValidationException(attributeName
- + " is not defined for the "
- + conf.getConfElement().getStramElement()
- + " or any of its child configurations.");
- }
-
- if (conf.getConfElement().getAmbiguousAttributes().contains(attributeName)) {
- //If the attribute name is ambiguous at this configuration level we should tell the user.
- LOG.warn("The attribute "
- + attributeName
- + " is ambiguous when specified on an " + conf.getConfElement().getStramElement());
- }
-
- if (conf.getConfElement().getContextAttributes().contains(attributeName)) {
- @SuppressWarnings("unchecked")
- Attribute<Object> attr = (Attribute<Object>)ContextUtils.CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(conf.getConfElement().getContextClass()).get(attributeName);
- conf.setAttribute(attr, propertyValue);
- } else {
- AttributeParseUtils.processAllConfsForAttribute(conf, attributeName, propertyValue);
- }
- } else {
- //This is a FQ attribute name
- Class<? extends Context> contextClass = AttributeParseUtils.getContainingContextClass(attributeName);
-
- //Convert to a simple name
- attributeName = AttributeParseUtils.getSimpleAttributeName(attributeName);
-
- if (!ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass).contains(attributeName)) {
- throw new ValidationException(attributeName + " is not a valid attribute in " + contextClass.getCanonicalName());
- }
+ parseAttributeElement(element, keys, index, conf, propertyValue, propertyName);
+ } else if ((element == StramElement.PROP) || ((element == null) && (conf.getDefaultChildElement() == StramElement.PROP))) {
+ parsePropertyElement(element, keys, index, conf, propertyValue, propertyName);
+ } else if (element != null) {
+ conf.parseElement(element, keys, index, propertyValue);
+ }
+ }
+ }
- ConfElement confWithAttr = ConfElement.CONTEXT_TO_CONF_ELEMENT.get(contextClass);
+ /**
+ * This is a helper method for {@link #parseStramPropertyTokens} which is responsible for parsing an app element.
+ * @param element The current {@link StramElement} of the property being parsed.
+ * @param keys The keys that the property being parsed was split into.
+ * @param index The current key that the parser is on.
+ * @param propertyValue The value associated with the property being parsed.
+ * @param propertyName The complete unprocessed name of the property being parsed.
+ */
+ private void parseAppElement(int index, String[] keys, StramElement element, Conf conf1, String propertyName, String propertyValue)
+ {
+ if ((index + 1) < keys.length) {
+ String name = keys[index+1];
+ Conf elConf = addConf(element, name, conf1);
+ if (elConf != null) {
+ parseStramPropertyTokens(keys, index + 2, propertyName, propertyValue, elConf);
+ } else {
+ LOG.error("Invalid configuration key: {}", propertyName);
+ }
+ } else {
+ LOG.warn("Invalid configuration key: {}", propertyName);
+ }
+ }
- conf = ConfElement.addConfs(conf, confWithAttr);
+ /**
+ * This is a helper method for {@link #parseStramPropertyTokens} which is responsible for parsing a gateway element.
+ * @param element The current {@link StramElement} of the property being parsed.
+ * @param keys The keys that the property being parsed was split into.
+ * @param index The current key that the parser is on.
+ * @param propertyValue The value associated with the property being parsed.
+ * @param propertyName The complete unprocessed name of the property being parsed.
+ */
+ private void parseGatewayElement(StramElement element, Conf conf1, String[] keys, int index, String propertyName, String propertyValue)
+ {
+ Conf elConf = addConf(element, null, conf1);
+ if (elConf != null) {
+ parseStramPropertyTokens(keys, index+1, propertyName, propertyValue, elConf);
+ } else {
+ LOG.error("Invalid configuration key: {}", propertyName);
+ }
+ }
- @SuppressWarnings("unchecked")
- Attribute<Object> attr = (Attribute<Object>)ContextUtils.CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(confWithAttr.getContextClass()).get(attributeName);
- conf.setAttribute(attr, propertyValue);
- }
- } else {
- LOG.error("Invalid configuration key: {}", propertyName);
+ /**
+ * This is a helper method for {@link #parseStramPropertyTokens} which is responsible for parsing an attribute.
+ * @param element The current {@link StramElement} of the property being parsed.
+ * @param keys The keys that the property being parsed was split into.
+ * @param index The current key that the parser is on.
+ * @param conf The current {@link Conf}.
+ * @param propertyValue The value associated with the property being parsed.
+ * @param propertyName The complete unprocessed name of the property being parsed.
+ */
+ private void parseAttributeElement(StramElement element, String[] keys, int index, Conf conf, String propertyValue, String propertyName)
+ {
+ String attributeName = AttributeParseUtils.getAttributeName(element, keys, index);
+ if (element != StramElement.ATTR) {
+ String expName = getCompleteKey(keys, 0, index) + KEY_SEPARATOR + StramElement.ATTR.getValue() + KEY_SEPARATOR + attributeName;
+ LOG.warn("Referencing the attribute as {} instead of {} is deprecated!", getCompleteKey(keys, 0), expName);
+ }
+ if (conf.getConfElement().getStramElement() == null) {
+ conf = addConf(StramElement.APPLICATION, WILDCARD, conf);
+ }
+ if (conf != null) {
+ if (AttributeParseUtils.isSimpleAttributeName(attributeName)) {
+ //The provided attribute name was a simple name
+ if (!AttributeParseUtils.ALL_SIMPLE_ATTRIBUTE_NAMES.contains(attributeName)) {
+ throw new ValidationException("Invalid attribute reference: " + getCompleteKey(keys, 0));
}
- } else if ((element == StramElement.PROP) || ((element == null) && (conf.getDefaultChildElement() == StramElement.PROP))) {
- // Currently opProps are only supported on operators and streams
- // Supporting current implementation where property can be directly specified under operator
- String prop;
- if (element == StramElement.PROP) {
- prop = getCompleteKey(keys, index+1);
- } else {
- prop = getCompleteKey(keys, index);
+ if (!conf.getConfElement().getAllChildAttributes().contains(attributeName)) {
+ throw new ValidationException(attributeName + " is not defined for the " + conf.getConfElement().getStramElement() + " or any of its child configurations.");
+ }
+ if (conf.getConfElement().getAmbiguousAttributes().contains(attributeName)) {
+ //If the attribute name is ambiguous at this configuration level we should tell the user.
+ LOG.warn("The attribute " + attributeName + " is ambiguous when specified on an " + conf.getConfElement().getStramElement());
}
- if (prop != null) {
- conf.setProperty(prop, propertyValue);
+ if (conf.getConfElement().getContextAttributes().contains(attributeName)) {
+ @SuppressWarnings(value = "unchecked")
+ Attribute<Object> attr = (Attribute<Object>)ContextUtils.CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(conf.getConfElement().getContextClass()).get(attributeName);
+ conf.setAttribute(attr, propertyValue);
} else {
- LOG.warn("Invalid property specification, no property name specified for {}", propertyName);
+ AttributeParseUtils.processAllConfsForAttribute(conf, attributeName, propertyValue);
}
- } else if (element != null) {
- conf.parseElement(element, keys, index, propertyValue);
+ } else {
+ //This is a FQ attribute name
+ Class<? extends Context> contextClass = AttributeParseUtils.getContainingContextClass(attributeName);
+ //Convert to a simple name
+ attributeName = AttributeParseUtils.getSimpleAttributeName(attributeName);
+ if (!ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass).contains(attributeName)) {
+ throw new ValidationException(attributeName + " is not a valid attribute in " + contextClass.getCanonicalName());
+ }
+ ConfElement confWithAttr = ConfElement.CONTEXT_TO_CONF_ELEMENT.get(contextClass);
+ conf = ConfElement.addConfs(conf, confWithAttr);
+ @SuppressWarnings("unchecked")
+ Attribute<Object> attr = (Attribute<Object>)ContextUtils.CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(confWithAttr.getContextClass()).get(attributeName);
+ conf.setAttribute(attr, propertyValue);
}
+ } else {
+ LOG.error("Invalid configuration key: {}", propertyName);
+ }
+ }
+
+ /**
+ * This is a helper method for {@link #parseStramPropertyTokens} which is responsible for parsing a prop.
+ * @param element The current {@link StramElement} of the property being parsed.
+ * @param keys The keys that the property being parsed was split into.
+ * @param index The current key that the parser is on.
+ * @param conf The current {@link Conf}.
+ * @param propertyValue The value associated with the property being parsed.
+ * @param propertyName The complete unprocessed name of the property being parsed.
+ */
+ private void parsePropertyElement(StramElement element, String[] keys, int index, Conf conf, String propertyValue, String propertyName)
+ {
+ // Currently opProps are only supported on operators and streams
+ // Supporting current implementation where property can be directly specified under operator
+ String prop;
+ if (element == StramElement.PROP) {
+ prop = getCompleteKey(keys, index+1);
+ } else {
+ prop = getCompleteKey(keys, index);
+ }
+ if (prop != null) {
+ conf.setProperty(prop, propertyValue);
+ } else {
+ LOG.warn("Invalid property specification, no property name specified for {}", propertyName);
}
}
@@ -1831,7 +1914,12 @@ public class LogicalPlanConfiguration {
* @return The completed key.
*/
private static String getCompleteKey(String[] keys, int start, int end) {
- StringBuilder sb = new StringBuilder(1024);
+ int length = 0;
+ for (int keyIndex = 0; keyIndex < keys.length; keyIndex++) {
+ length += keys[keyIndex].length();
+ }
+
+ StringBuilder sb = new StringBuilder(length);
for (int i = start; i < end; ++i) {
if (i > start) {
sb.append(KEY_SEPARATOR);
[5/8] incubator-apex-core git commit: APEX-28 #resolve
Posted by cs...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java b/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java
deleted file mode 100644
index 78173d8..0000000
--- a/engine/src/test/java/com/datatorrent/stram/plan/LogicalPlanTest.java
+++ /dev/null
@@ -1,990 +0,0 @@
-/**
- * Copyright (C) 2015 DataTorrent, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.datatorrent.stram.plan;
-
-import com.datatorrent.common.util.BaseOperator;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.Serializable;
-
-import java.util.*;
-
-import javax.validation.*;
-import javax.validation.constraints.AssertTrue;
-import javax.validation.constraints.Min;
-import javax.validation.constraints.NotNull;
-import javax.validation.constraints.Pattern;
-
-import com.esotericsoftware.kryo.DefaultSerializer;
-import com.esotericsoftware.kryo.serializers.JavaSerializer;
-import com.google.common.collect.Maps;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-import com.datatorrent.common.partitioner.StatelessPartitioner;
-import com.datatorrent.api.*;
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.Context.PortContext;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.annotation.InputPortFieldAnnotation;
-import com.datatorrent.api.annotation.OperatorAnnotation;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.netlet.util.Slice;
-import com.datatorrent.stram.engine.GenericTestOperator;
-import com.datatorrent.stram.engine.TestGeneratorInputOperator;
-import com.datatorrent.stram.engine.TestNonOptionalOutportInputOperator;
-import com.datatorrent.stram.engine.TestOutputOperator;
-import com.datatorrent.stram.plan.logical.LogicalPlan;
-import com.datatorrent.stram.plan.logical.LogicalPlan;
-import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
-import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
-import com.datatorrent.stram.support.StramTestSupport.MemoryStorageAgent;
-import com.datatorrent.stram.support.StramTestSupport.RegexMatcher;
-
-public class LogicalPlanTest {
-
- @Test
- public void testCycleDetection() {
- LogicalPlan dag = new LogicalPlan();
-
- //NodeConf operator1 = b.getOrAddNode("operator1");
- GenericTestOperator operator2 = dag.addOperator("operator2", GenericTestOperator.class);
- GenericTestOperator operator3 = dag.addOperator("operator3", GenericTestOperator.class);
- GenericTestOperator operator4 = dag.addOperator("operator4", GenericTestOperator.class);
- //NodeConf operator5 = b.getOrAddNode("operator5");
- //NodeConf operator6 = b.getOrAddNode("operator6");
- GenericTestOperator operator7 = dag.addOperator("operator7", GenericTestOperator.class);
-
- // strongly connect n2-n3-n4-n2
- dag.addStream("n2n3", operator2.outport1, operator3.inport1);
-
- dag.addStream("n3n4", operator3.outport1, operator4.inport1);
-
- dag.addStream("n4n2", operator4.outport1, operator2.inport1);
-
- // self referencing operator cycle
- StreamMeta n7n7 = dag.addStream("n7n7", operator7.outport1, operator7.inport1);
- try {
- n7n7.addSink(operator7.inport1);
- fail("cannot add to stream again");
- } catch (Exception e) {
- // expected, stream can have single input/output only
- }
-
- List<List<String>> cycles = new ArrayList<List<String>>();
- dag.findStronglyConnected(dag.getMeta(operator7), cycles);
- assertEquals("operator self reference", 1, cycles.size());
- assertEquals("operator self reference", 1, cycles.get(0).size());
- assertEquals("operator self reference", dag.getMeta(operator7).getName(), cycles.get(0).get(0));
-
- // 3 operator cycle
- cycles.clear();
- dag.findStronglyConnected(dag.getMeta(operator4), cycles);
- assertEquals("3 operator cycle", 1, cycles.size());
- assertEquals("3 operator cycle", 3, cycles.get(0).size());
- assertTrue("operator2", cycles.get(0).contains(dag.getMeta(operator2).getName()));
- assertTrue("operator3", cycles.get(0).contains(dag.getMeta(operator3).getName()));
- assertTrue("operator4", cycles.get(0).contains(dag.getMeta(operator4).getName()));
-
- try {
- dag.validate();
- fail("validation should fail");
- } catch (ValidationException e) {
- // expected
- }
-
- }
-
- public static class ValidationOperator extends BaseOperator {
- public final transient DefaultOutputPort<Object> goodOutputPort = new DefaultOutputPort<Object>();
-
- public final transient DefaultOutputPort<Object> badOutputPort = new DefaultOutputPort<Object>();
- }
-
- public static class CounterOperator extends BaseOperator {
- final public transient InputPort<Object> countInputPort = new DefaultInputPort<Object>() {
- @Override
- final public void process(Object payload) {
- }
- };
- }
-
- @Test
- public void testLogicalPlanSerialization() throws Exception {
-
- LogicalPlan dag = new LogicalPlan();
- dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
-
- ValidationOperator validationNode = dag.addOperator("validationNode", ValidationOperator.class);
- CounterOperator countGoodNode = dag.addOperator("countGoodNode", CounterOperator.class);
- CounterOperator countBadNode = dag.addOperator("countBadNode", CounterOperator.class);
- //ConsoleOutputOperator echoBadNode = dag.addOperator("echoBadNode", ConsoleOutputOperator.class);
-
- // good tuples to counter operator
- dag.addStream("goodTuplesStream", validationNode.goodOutputPort, countGoodNode.countInputPort);
-
- // bad tuples to separate stream and echo operator
- // (stream with 2 outputs)
- dag.addStream("badTuplesStream", validationNode.badOutputPort, countBadNode.countInputPort);
-
- Assert.assertEquals("number root operators", 1, dag.getRootOperators().size());
- Assert.assertEquals("root operator id", "validationNode", dag.getRootOperators().get(0).getName());
-
- dag.getContextAttributes(countGoodNode).put(OperatorContext.SPIN_MILLIS, 10);
-
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- LogicalPlan.write(dag, bos);
-
- // System.out.println("serialized size: " + bos.toByteArray().length);
-
- ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
- LogicalPlan dagClone = LogicalPlan.read(bis);
- Assert.assertNotNull(dagClone);
- Assert.assertEquals("number operators in clone", dag.getAllOperators().size(), dagClone.getAllOperators().size());
- Assert.assertEquals("number root operators in clone", 1, dagClone.getRootOperators().size());
- Assert.assertTrue("root operator in operators", dagClone.getAllOperators().contains(dagClone.getRootOperators().get(0)));
-
-
- Operator countGoodNodeClone = dagClone.getOperatorMeta("countGoodNode").getOperator();
- Assert.assertEquals("", new Integer(10), dagClone.getContextAttributes(countGoodNodeClone).get(OperatorContext.SPIN_MILLIS));
-
- }
-
- @Test
- public void testDeleteOperator()
- {
- LogicalPlan dag = new LogicalPlan();
- TestGeneratorInputOperator input = dag.addOperator("input1", TestGeneratorInputOperator.class);
- GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
- GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
- dag.addStream("s0", input.outport, o1.inport1);
- StreamMeta s1 = dag.addStream("s1", o1.outport1, o2.inport1);
- dag.validate();
- Assert.assertEquals("", 3, dag.getAllOperators().size());
-
- dag.removeOperator(o2);
- s1.remove();
- dag.validate();
- Assert.assertEquals("", 2, dag.getAllOperators().size());
- }
-
- public static class ValidationTestOperator extends BaseOperator implements InputOperator {
- @NotNull
- @Pattern(regexp=".*malhar.*", message="Value has to contain 'malhar'!")
- private String stringField1;
-
- @Min(2)
- private int intField1;
-
- @AssertTrue(message="stringField1 should end with intField1")
- private boolean isValidConfiguration() {
- return stringField1.endsWith(String.valueOf(intField1));
- }
-
- private String getterProperty2 = "";
-
- @NotNull
- public String getProperty2() {
- return getterProperty2;
- }
-
- public void setProperty2(String s) {
- // annotations need to be on the getter
- getterProperty2 = s;
- }
-
- private String[] stringArrayField;
-
- public String[] getStringArrayField() {
- return stringArrayField;
- }
-
- public void setStringArrayField(String[] stringArrayField) {
- this.stringArrayField = stringArrayField;
- }
-
- public class Nested {
- @NotNull
- private String property = "";
-
- public String getProperty() {
- return property;
- }
-
- public void setProperty(String property) {
- this.property = property;
- }
-
- }
-
- @Valid
- private final Nested nestedBean = new Nested();
-
- private String stringProperty2;
-
- public String getStringProperty2() {
- return stringProperty2;
- }
-
- public void setStringProperty2(String stringProperty2) {
- this.stringProperty2 = stringProperty2;
- }
-
- private Map<String, String> mapProperty = Maps.newHashMap();
-
- public Map<String, String> getMapProperty()
- {
- return mapProperty;
- }
-
- public void setMapProperty(Map<String, String> mapProperty)
- {
- this.mapProperty = mapProperty;
- }
-
- @Override
- public void emitTuples() {
- // Emit no tuples
-
- }
-
- }
-
- @Test
- public void testOperatorValidation() {
-
- ValidationTestOperator bean = new ValidationTestOperator();
- bean.stringField1 = "malhar1";
- bean.intField1 = 1;
-
- // ensure validation standalone produces expected results
- ValidatorFactory factory =
- Validation.buildDefaultValidatorFactory();
- Validator validator = factory.getValidator();
- Set<ConstraintViolation<ValidationTestOperator>> constraintViolations =
- validator.validate(bean);
- //for (ConstraintViolation<ValidationTestOperator> cv : constraintViolations) {
- // System.out.println("validation error: " + cv);
- //}
- Assert.assertEquals("" + constraintViolations,1, constraintViolations.size());
- ConstraintViolation<ValidationTestOperator> cv = constraintViolations.iterator().next();
- Assert.assertEquals("", bean.intField1, cv.getInvalidValue());
- Assert.assertEquals("", "intField1", cv.getPropertyPath().toString());
-
- // ensure DAG validation produces matching results
- LogicalPlan dag = new LogicalPlan();
- bean = dag.addOperator("testOperator", bean);
-
- try {
- dag.validate();
- Assert.fail("should throw ConstraintViolationException");
- } catch (ConstraintViolationException e) {
- Assert.assertEquals("violation details", constraintViolations, e.getConstraintViolations());
- String expRegex = ".*ValidationTestOperator\\{name=null}, propertyPath='intField1', message='must be greater than or equal to 2',.*value=1}]";
- Assert.assertThat("exception message", e.getMessage(), RegexMatcher.matches(expRegex));
- }
-
- try {
- bean.intField1 = 3;
- dag.validate();
- Assert.fail("should throw ConstraintViolationException");
- } catch (ConstraintViolationException e) {
- ConstraintViolation<?> cv2 = e.getConstraintViolations().iterator().next();
- Assert.assertEquals("" + e.getConstraintViolations(), 1, constraintViolations.size());
- Assert.assertEquals("", false, cv2.getInvalidValue());
- Assert.assertEquals("", "validConfiguration", cv2.getPropertyPath().toString());
- }
- bean.stringField1 = "malhar3";
-
- // annotated getter
- try {
- bean.getterProperty2 = null;
- dag.validate();
- Assert.fail("should throw ConstraintViolationException");
- } catch (ConstraintViolationException e) {
- ConstraintViolation<?> cv2 = e.getConstraintViolations().iterator().next();
- Assert.assertEquals("" + e.getConstraintViolations(), 1, constraintViolations.size());
- Assert.assertEquals("", null, cv2.getInvalidValue());
- Assert.assertEquals("", "property2", cv2.getPropertyPath().toString());
- }
- bean.getterProperty2 = "";
-
- // nested property
- try {
- bean.nestedBean.property = null;
- dag.validate();
- Assert.fail("should throw ConstraintViolationException");
- } catch (ConstraintViolationException e) {
- ConstraintViolation<?> cv2 = e.getConstraintViolations().iterator().next();
- Assert.assertEquals("" + e.getConstraintViolations(), 1, constraintViolations.size());
- Assert.assertEquals("", null, cv2.getInvalidValue());
- Assert.assertEquals("", "nestedBean.property", cv2.getPropertyPath().toString());
- }
- bean.nestedBean.property = "";
-
- // all valid
- dag.validate();
-
- }
-
- @OperatorAnnotation(partitionable = false)
- public static class TestOperatorAnnotationOperator extends BaseOperator {
-
- @InputPortFieldAnnotation( optional = true)
- final public transient DefaultInputPort<Object> input1 = new DefaultInputPort<Object>() {
- @Override
- public void process(Object tuple) {
- }
- };
- }
-
- class NoInputPortOperator extends BaseOperator {
- }
-
- @Test
- public void testValidationForNonInputRootOperator() {
- LogicalPlan dag = new LogicalPlan();
- NoInputPortOperator x = dag.addOperator("x", new NoInputPortOperator());
- try {
- dag.validate();
- Assert.fail("should fail because root operator is not input operator");
- } catch (ValidationException e) {
- // expected
- }
- }
-
- @OperatorAnnotation(partitionable = false)
- public static class TestOperatorAnnotationOperator2 extends BaseOperator implements Partitioner<TestOperatorAnnotationOperator2> {
-
- @Override
- public Collection<Partition<TestOperatorAnnotationOperator2>> definePartitions(Collection<Partition<TestOperatorAnnotationOperator2>> partitions, PartitioningContext context)
- {
- return null;
- }
-
- @Override
- public void partitioned(Map<Integer, Partition<TestOperatorAnnotationOperator2>> partitions)
- {
- }
- }
-
- @Test
- public void testOperatorAnnotation() {
- LogicalPlan dag = new LogicalPlan();
- TestGeneratorInputOperator input = dag.addOperator("input1", TestGeneratorInputOperator.class);
- TestOperatorAnnotationOperator operator = dag.addOperator("operator1", TestOperatorAnnotationOperator.class);
- dag.addStream("Connection", input.outport, operator.input1);
-
-
- dag.setAttribute(operator, OperatorContext.PARTITIONER, new StatelessPartitioner<TestOperatorAnnotationOperator>(2));
-
- try {
- dag.validate();
- Assert.fail("should raise operator is not partitionable for operator1");
- } catch (ValidationException e) {
- Assert.assertEquals("", "Operator " + dag.getMeta(operator).getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!", e.getMessage());
- }
-
- dag.setAttribute(operator, OperatorContext.PARTITIONER, null);
- dag.setInputPortAttribute(operator.input1, PortContext.PARTITION_PARALLEL, true);
-
- try {
- dag.validate();
- Assert.fail("should raise operator is not partitionable for operator1");
- } catch (ValidationException e) {
- Assert.assertEquals("", "Operator " + dag.getMeta(operator).getName() + " is not partitionable but PARTITION_PARALLEL attribute is set", e.getMessage());
- }
-
- dag.setInputPortAttribute(operator.input1, PortContext.PARTITION_PARALLEL, false);
- dag.validate();
-
- dag.removeOperator(operator);
- TestOperatorAnnotationOperator2 operator2 = dag.addOperator("operator2", TestOperatorAnnotationOperator2.class);
-
- try {
- dag.validate();
- Assert.fail("should raise operator is not partitionable for operator2");
- } catch (ValidationException e) {
- Assert.assertEquals("Operator " + dag.getMeta(operator2).getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!", e.getMessage());
- }
- }
-
- @Test
- public void testPortConnectionValidation() {
-
- LogicalPlan dag = new LogicalPlan();
-
- TestNonOptionalOutportInputOperator input = dag.addOperator("input1", TestNonOptionalOutportInputOperator.class);
-
- try {
- dag.validate();
- Assert.fail("should raise port not connected for input1.outputPort1");
-
- } catch (ValidationException e) {
- Assert.assertEquals("", "Output port connection required: input1.outport1", e.getMessage());
- }
-
- GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
- dag.addStream("stream1", input.outport1, o1.inport1);
- dag.validate();
-
- // required input
- dag.addOperator("counter", CounterOperator.class);
- try {
- dag.validate();
- } catch (ValidationException e) {
- Assert.assertEquals("", "Input port connection required: counter.countInputPort", e.getMessage());
- }
-
- }
-
- @Test
- public void testAtMostOnceProcessingModeValidation() {
- LogicalPlan dag = new LogicalPlan();
-
- TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
- TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
-
- GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class);
- dag.setAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
-
- dag.addStream("input1.outport", input1.outport, amoOper.inport1);
- dag.addStream("input2.outport", input2.outport, amoOper.inport2);
-
- GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class);
- dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE);
- dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1);
-
- try {
- dag.validate();
- Assert.fail("Exception expected for " + outputOper);
- } catch (ValidationException ve) {
- Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/AT_MOST_ONCE");
- }
- dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, null);
- dag.validate();
-
- OperatorMeta outputOperOm = dag.getMeta(outputOper);
- Assert.assertEquals("" + outputOperOm.getAttributes(), Operator.ProcessingMode.AT_MOST_ONCE, outputOperOm.getValue(OperatorContext.PROCESSING_MODE));
-
- }
-
- @Test
- public void testExactlyOnceProcessingModeValidation() {
- LogicalPlan dag = new LogicalPlan();
-
- TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
- TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
-
- GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class);
- dag.setAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.EXACTLY_ONCE);
-
- dag.addStream("input1.outport", input1.outport, amoOper.inport1);
- dag.addStream("input2.outport", input2.outport, amoOper.inport2);
-
- GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class);
- dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1);
-
- try {
- dag.validate();
- Assert.fail("Exception expected for " + outputOper);
- } catch (ValidationException ve) {
- Assert.assertEquals("", ve.getMessage(), "Processing mode for outputOper should be AT_MOST_ONCE for source amoOper/EXACTLY_ONCE");
- }
-
- dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE);
-
- try {
- dag.validate();
- Assert.fail("Exception expected for " + outputOper);
- } catch (ValidationException ve) {
- Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/EXACTLY_ONCE");
- }
-
- // AT_MOST_ONCE is valid
- dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
- dag.validate();
- }
-
- @Test
- public void testLocalityValidation() {
- LogicalPlan dag = new LogicalPlan();
-
- TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
- GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
- StreamMeta s1 = dag.addStream("input1.outport", input1.outport, o1.inport1).setLocality(Locality.THREAD_LOCAL);
- dag.validate();
-
- TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
- dag.addStream("input2.outport", input2.outport, o1.inport2);
-
- try {
- dag.validate();
- Assert.fail("Exception expected for " + o1);
- } catch (ValidationException ve) {
- Assert.assertThat("", ve.getMessage(), RegexMatcher.matches("Locality THREAD_LOCAL invalid for operator .* with multiple input streams .*"));
- }
-
- s1.setLocality(null);
- dag.validate();
- }
-
- private class TestAnnotationsOperator extends BaseOperator implements InputOperator {
- //final public transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
-
- @OutputPortFieldAnnotation( optional=false)
- final public transient DefaultOutputPort<Object> outport2 = new DefaultOutputPort<Object>();
-
- @Override
- public void emitTuples() {
- // Emit Nothing
-
- }
- }
-
- private class TestAnnotationsOperator2 extends BaseOperator implements InputOperator{
- // multiple ports w/o annotation, one of them must be connected
- final public transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
-
- @Override
- public void emitTuples() {
- // Emit Nothing
-
- }
- }
-
- private class TestAnnotationsOperator3 extends BaseOperator implements InputOperator{
- // multiple ports w/o annotation, one of them must be connected
- @OutputPortFieldAnnotation( optional=true)
- final public transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
- @OutputPortFieldAnnotation( optional=true)
- final public transient DefaultOutputPort<Object> outport2 = new DefaultOutputPort<Object>();
- @Override
- public void emitTuples() {
- // Emit Nothing
-
- }
- }
-
- @Test
- public void testOutputPortAnnotation() {
- LogicalPlan dag = new LogicalPlan();
- TestAnnotationsOperator ta1 = dag.addOperator("testAnnotationsOperator", new TestAnnotationsOperator());
-
- try {
- dag.validate();
- Assert.fail("should raise: port connection required");
- } catch (ValidationException e) {
- Assert.assertEquals("", "Output port connection required: testAnnotationsOperator.outport2", e.getMessage());
- }
-
- TestOutputOperator o2 = dag.addOperator("sink", new TestOutputOperator());
- dag.addStream("s1", ta1.outport2, o2.inport);
-
- dag.validate();
-
- TestAnnotationsOperator2 ta2 = dag.addOperator("multiOutputPorts1", new TestAnnotationsOperator2());
-
- try {
- dag.validate();
- Assert.fail("should raise: At least one output port must be connected");
- } catch (ValidationException e) {
- Assert.assertEquals("", "At least one output port must be connected: multiOutputPorts1", e.getMessage());
- }
- TestOutputOperator o3 = dag.addOperator("o3", new TestOutputOperator());
- dag.addStream("s2", ta2.outport1, o3.inport);
-
- dag.addOperator("multiOutputPorts3", new TestAnnotationsOperator3());
- dag.validate();
-
- }
-
- /**
- * Operator that can be used with default Java serialization instead of Kryo
- */
- @DefaultSerializer(JavaSerializer.class)
- public static class JdkSerializableOperator extends BaseOperator implements Serializable {
- private static final long serialVersionUID = -4024202339520027097L;
-
- public abstract class SerializableInputPort<T> implements InputPort<T>, Sink<T>, java.io.Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Sink<T> getSink() {
- return this;
- }
-
- @Override
- public void setConnected(boolean connected) {
- }
-
- @Override
- public void setup(PortContext context)
- {
- }
-
- @Override
- public void teardown()
- {
- }
-
- @Override
- public StreamCodec<T> getStreamCodec() {
- return null;
- }
- }
-
- @InputPortFieldAnnotation( optional=true)
- final public InputPort<Object> inport1 = new SerializableInputPort<Object>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- final public void put(Object payload)
- {
- }
-
- @Override
- public int getCount(boolean reset)
- {
- return 0;
- }
-
- };
- }
-
- @Test
- public void testJdkSerializableOperator() throws Exception {
- LogicalPlan dag = new LogicalPlan();
- dag.addOperator("o1", new JdkSerializableOperator());
-
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- LogicalPlan.write(dag, outStream);
- outStream.close();
-
- LogicalPlan clonedDag = LogicalPlan.read(new ByteArrayInputStream(outStream.toByteArray()));
- JdkSerializableOperator o1Clone = (JdkSerializableOperator)clonedDag.getOperatorMeta("o1").getOperator();
- Assert.assertNotNull("port object null", o1Clone.inport1);
- }
-
- private static class TestStreamCodec implements StreamCodec<Object> {
- @Override
- public Object fromByteArray(Slice fragment)
- {
- return fragment.stringValue();
- }
-
- @Override
- public Slice toByteArray(Object o)
- {
- byte[] b = o.toString().getBytes();
- return new Slice(b, 0, b.length);
- }
-
- @Override
- public int getPartition(Object o)
- {
- return o.hashCode() / 2;
- }
- }
-
- public static class TestPortCodecOperator extends BaseOperator {
- public transient final DefaultInputPort<Object> inport1 = new DefaultInputPort<Object>()
- {
- @Override
- public void process(Object tuple)
- {
-
- }
-
- @Override
- public StreamCodec<Object> getStreamCodec()
- {
- return new TestStreamCodec();
- }
- };
-
- @OutputPortFieldAnnotation( optional = true)
- public transient final DefaultOutputPort<Object> outport = new DefaultOutputPort<Object>();
- }
-
- /*
- @Test
- public void testStreamCodec() throws Exception {
- LogicalPlan dag = new LogicalPlan();
- TestGeneratorInputOperator input = dag.addOperator("input", TestGeneratorInputOperator.class);
- GenericTestOperator gto1 = dag.addOperator("gto1", GenericTestOperator.class);
- StreamMeta stream1 = dag.addStream("s1", input.outport, gto1.inport1);
- StreamCodec<?> codec1 = new TestStreamCodec();
- dag.setInputPortAttribute(gto1.inport1, PortContext.STREAM_CODEC, codec1);
- dag.validate();
- //Assert.assertEquals("Stream codec not set", stream1.getStreamCodec(), codec1);
-
- GenericTestOperator gto2 = dag.addOperator("gto2", GenericTestOperator.class);
- GenericTestOperator gto3 = dag.addOperator("gto3", GenericTestOperator.class);
- StreamMeta stream2 = dag.addStream("s2", gto1.outport1, gto2.inport1, gto3.inport1);
- dag.setInputPortAttribute(gto2.inport1, PortContext.STREAM_CODEC, codec1);
- try {
- dag.validate();
- } catch (ValidationException e) {
- String msg = e.getMessage();
- if (!msg.startsWith("Stream codec not set on input port") || !msg.contains("gto3")
- || !msg.contains(codec1.toString()) || !msg.endsWith("was specified on another port")) {
- Assert.fail(String.format("LogicalPlan validation error msg: %s", msg));
- }
- }
-
- dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec1);
- dag.validate();
- //Assert.assertEquals("Stream codec not set", stream2.getStreamCodec(), codec1);
-
- StreamCodec<?> codec2 = new TestStreamCodec();
- dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec2);
- try {
- dag.validate();
- } catch (ValidationException e) {
- String msg = e.getMessage();
- if (!msg.startsWith("Conflicting stream codec set on input port") || !msg.contains("gto3")
- || !msg.contains(codec2.toString()) || !msg.endsWith("was specified on another port")) {
- Assert.fail(String.format("LogicalPlan validation error msg: %s", msg));
- }
- }
-
- dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec1);
- TestPortCodecOperator pco = dag.addOperator("pco", TestPortCodecOperator.class);
- StreamMeta stream3 = dag.addStream("s3", gto2.outport1, pco.inport1);
- dag.validate();
- //Assert.assertEquals("Stream codec class not set", stream3.getCodecClass(), TestStreamCodec.class);
-
- dag.setInputPortAttribute(pco.inport1, PortContext.STREAM_CODEC, codec2);
- dag.validate();
- //Assert.assertEquals("Stream codec not set", stream3.getStreamCodec(), codec2);
- }
- */
-
- @Test
- public void testCheckpointableWithinAppWindowAnnotation()
- {
- LogicalPlan dag = new LogicalPlan();
- TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
- GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
- dag.addStream("Stream1", input1.outport, x.inport1);
- dag.setAttribute(x, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
- dag.setAttribute(x, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
- dag.validate();
-
- TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
- CheckpointableWithinAppWindowOperator y = dag.addOperator("y", new CheckpointableWithinAppWindowOperator());
- dag.addStream("Stream2", input2.outport, y.inport1);
- dag.setAttribute(y, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
- dag.setAttribute(y, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
- dag.validate();
-
- TestGeneratorInputOperator input3 = dag.addOperator("input3", TestGeneratorInputOperator.class);
- NotCheckpointableWithinAppWindowOperator z = dag.addOperator("z", new NotCheckpointableWithinAppWindowOperator());
- dag.addStream("Stream3", input3.outport, z.inport1);
- dag.setAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
- dag.setAttribute(z, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
- try {
- dag.validate();
- Assert.fail("should fail because chekpoint window count is not a factor of application window count");
- }
- catch (ValidationException e) {
- // expected
- }
-
- dag.setAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 30);
- dag.validate();
-
- dag.setAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 45);
- try {
- dag.validate();
- Assert.fail("should fail because chekpoint window count is not a factor of application window count");
- }
- catch (ValidationException e) {
- // expected
- }
- }
-
- @OperatorAnnotation(checkpointableWithinAppWindow = true)
- class CheckpointableWithinAppWindowOperator extends GenericTestOperator
- {
- }
-
- @OperatorAnnotation(checkpointableWithinAppWindow = false)
- class NotCheckpointableWithinAppWindowOperator extends GenericTestOperator
- {
- }
-
- @Test
- public void testInputPortHiding()
- {
- LogicalPlan dag = new LogicalPlan();
- TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
- Operator2 operator2 = dag.addOperator("operator2", new Operator2());
- dag.addStream("Stream1", input1.outport, operator2.input);
- dag.validate();
- }
-
- @Test
- public void testInvalidInputPortConnection()
- {
- LogicalPlan dag = new LogicalPlan();
- TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
- Operator1 operator1 = dag.addOperator("operator3", new Operator3());
- dag.addStream("Stream1", input1.outport, operator1.input);
- try {
- dag.validate();
- } catch (ValidationException ex) {
- Assert.assertTrue("validation message", ex.getMessage().startsWith("Invalid port connected"));
- return;
- }
- Assert.fail();
- }
-
- class Operator1 extends BaseOperator
- {
- public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
- {
- @Override
- public void process(Object tuple)
- {
-
- }
- };
- }
-
- class Operator2 extends Operator1
- {
- public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
- {
- @Override
- public void process(Object tuple)
- {
-
- }
- };
- }
-
- class Operator3 extends Operator1
- {
- @InputPortFieldAnnotation(optional = true)
- public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
- {
- @Override
- public void process(Object tuple)
- {
-
- }
- };
- }
-
- @Test
- public void testOutputPortHiding()
- {
- LogicalPlan dag = new LogicalPlan();
- Operator5 operator5 = dag.addOperator("input", new Operator5());
- Operator2 operator2 = dag.addOperator("operator2", new Operator2());
- dag.addStream("Stream1", operator5.output, operator2.input);
- dag.validate();
- }
-
- @Test(expected = ValidationException.class)
- public void testInvalidOutputPortConnection()
- {
- LogicalPlan dag = new LogicalPlan();
- Operator4 operator4 = dag.addOperator("input", new Operator5());
- Operator3 operator3 = dag.addOperator("operator3", new Operator3());
- dag.addStream("Stream1", operator4.output, operator3.input);
- dag.validate();
- }
-
- class Operator4 extends BaseOperator implements InputOperator
- {
- public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
-
- @Override
- public void emitTuples()
- {
-
- }
- }
-
- class Operator5 extends Operator4
- {
- public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
- }
-
- /*
- These were tests for operator semantics that verified if an operator class implements InputOperator then the same class should not declare input ports.
- This would be done later when we are able to verify user code at compile-time.
-
- validation()
- {
- if (n.getOperator() instanceof InputOperator) {
- try {
- for (Class<?> clazz : n.getOperator().getClass().getInterfaces()) {
- if (clazz.getName().equals(InputOperator.class.getName())) {
- for (Field field : n.getOperator().getClass().getDeclaredFields()) {
- field.setAccessible(true);
- Object declaredObject = field.get(n.getOperator());
- if (declaredObject instanceof InputPort) {
- throw new ValidationException("Operator class implements InputOperator and also declares input ports: " + n.name);
- }
- }
- break;
- }
- }
- }
- catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
- }
- @Test
- public void testInvalidInputOperatorDeclaration()
- {
- LogicalPlan dag = new LogicalPlan();
-
- TestGeneratorInputOperator.InvalidInputOperator inputOperator = dag.addOperator("input", new TestGeneratorInputOperator.InvalidInputOperator());
- GenericTestOperator operator2 = dag.addOperator("operator2", GenericTestOperator.class);
-
- dag.addStream("stream1", inputOperator.outport, operator2.inport1);
-
- try {
- dag.validate();
- fail("validation should fail");
- }
- catch (ValidationException e) {
- // expected
- }
- }
-
- @Test
- public void testValidInputOperatorDeclaration()
- {
- LogicalPlan dag = new LogicalPlan();
-
- TestGeneratorInputOperator.ValidGenericOperator operator1 = dag.addOperator("input", new TestGeneratorInputOperator.ValidGenericOperator());
- GenericTestOperator operator2 = dag.addOperator("operator2", GenericTestOperator.class);
-
- dag.addStream("stream1", operator1.outport, operator2.inport1);
- dag.validate();
- }
- */
-}
[4/8] incubator-apex-core git commit: APEX-28 #resolve
Posted by cs...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanConfigurationTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanConfigurationTest.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanConfigurationTest.java
new file mode 100644
index 0000000..077e3a9
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanConfigurationTest.java
@@ -0,0 +1,1511 @@
+/**
+ * Copyright (C) 2015 DataTorrent, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datatorrent.stram.plan.logical;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.lang.reflect.Field;
+
+import java.util.*;
+
+import javax.validation.ValidationException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.*;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.*;
+import com.datatorrent.api.Attribute.AttributeMap;
+import com.datatorrent.api.Attribute.AttributeMap.AttributeInitializer;
+import com.datatorrent.api.Context.DAGContext;
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.StringCodec.Integer2String;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.common.codec.JsonStreamCodec;
+import com.datatorrent.common.util.BasicContainerOptConfigurator;
+import com.datatorrent.stram.PartitioningTest.PartitionLoadWatch;
+import com.datatorrent.stram.client.StramClientUtils;
+import com.datatorrent.stram.engine.GenericTestOperator;
+import com.datatorrent.stram.engine.TestGeneratorInputOperator;
+import com.datatorrent.stram.plan.SchemaTestOperator;
+import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.AttributeParseUtils;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.ConfElement;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.ContextUtils;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.StramElement;
+import com.datatorrent.stram.plan.logical.LogicalPlanTest.ValidationTestOperator;
+import com.datatorrent.stram.support.StramTestSupport.RegexMatcher;
+
+public class LogicalPlanConfigurationTest {
+
+ static {
+ @SuppressWarnings("MismatchedReadAndWriteOfArray")
+ Object serial[] = new Object[] {MockContext1.serialVersionUID, MockContext2.serialVersionUID};
+ }
+
+ private static OperatorMeta assertNode(LogicalPlan dag, String id) {
+ OperatorMeta n = dag.getOperatorMeta(id);
+ assertNotNull("operator exists id=" + id, n);
+ return n;
+ }
+
+ /**
+ * Test read from dt-site.xml in Hadoop configuration format.
+ */
+ @Test
+ public void testLoadFromConfigXml() {
+ Configuration conf = new Configuration(false);
+ conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
+
+ LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
+
+ LogicalPlan dag = new LogicalPlan();
+ builder.populateDAG(dag);
+ dag.validate();
+
+ assertEquals("number of operator confs", 6, dag.getAllOperators().size());
+
+ OperatorMeta operator1 = assertNode(dag, "operator1");
+ OperatorMeta operator2 = assertNode(dag, "operator2");
+ OperatorMeta operator3 = assertNode(dag, "operator3");
+ OperatorMeta operator4 = assertNode(dag, "operator4");
+
+ assertNotNull("operatorConf for root", operator1);
+ assertEquals("operatorId set", "operator1", operator1.getName());
+
+ // verify operator instantiation
+ assertEquals(operator1.getOperator().getClass(), TestGeneratorInputOperator.class);
+ TestGeneratorInputOperator GenericTestNode = (TestGeneratorInputOperator)operator1.getOperator();
+ assertEquals("myStringPropertyValue", GenericTestNode.getMyStringProperty());
+
+ // check links
+ assertEquals("operator1 inputs", 0, operator1.getInputStreams().size());
+ assertEquals("operator1 outputs", 1, operator1.getOutputStreams().size());
+ StreamMeta n1n2 = operator2.getInputStreams().get(operator2.getMeta(((GenericTestOperator)operator2.getOperator()).inport1));
+ assertNotNull("n1n2", n1n2);
+
+ // output/input stream object same
+ assertEquals("rootNode out is operator2 in", n1n2, operator1.getOutputStreams().get(operator1.getMeta(((TestGeneratorInputOperator)operator1.getOperator()).outport)));
+ assertEquals("n1n2 source", operator1, n1n2.getSource().getOperatorMeta());
+ Assert.assertEquals("n1n2 targets", 1, n1n2.getSinks().size());
+ Assert.assertEquals("n1n2 target", operator2, n1n2.getSinks().get(0).getOperatorWrapper());
+
+ assertEquals("stream name", "n1n2", n1n2.getName());
+ Assert.assertEquals("n1n2 not inline (default)", null, n1n2.getLocality());
+
+ // operator 2 streams to operator 3 and operator 4
+ assertEquals("operator 2 number of outputs", 1, operator2.getOutputStreams().size());
+ StreamMeta fromNode2 = operator2.getOutputStreams().values().iterator().next();
+
+ Set<OperatorMeta> targetNodes = Sets.newHashSet();
+ for (LogicalPlan.InputPortMeta ip : fromNode2.getSinks()) {
+ targetNodes.add(ip.getOperatorWrapper());
+ }
+ Assert.assertEquals("outputs " + fromNode2, Sets.newHashSet(operator3, operator4), targetNodes);
+
+ OperatorMeta operator6 = assertNode(dag, "operator6");
+
+ List<OperatorMeta> rootNodes = dag.getRootOperators();
+ assertEquals("number root operators", 2, rootNodes.size());
+ assertTrue("root operator2", rootNodes.contains(operator1));
+ assertTrue("root operator6", rootNodes.contains(operator6));
+
+ for (OperatorMeta n : rootNodes) {
+ printTopology(n, dag, 0);
+ }
+
+ }
+
+ private void printTopology(OperatorMeta operator, DAG tplg, int level) {
+ String prefix = "";
+ if (level > 0) {
+ prefix = StringUtils.repeat(" ", 20*(level-1)) + " |" + StringUtils.repeat("-", 17);
+ }
+ logger.debug(prefix + operator.getName());
+ for (StreamMeta downStream : operator.getOutputStreams().values()) {
+ if (!downStream.getSinks().isEmpty()) {
+ for (LogicalPlan.InputPortMeta targetNode : downStream.getSinks()) {
+ printTopology(targetNode.getOperatorWrapper(), tplg, level+1);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testLoadFromPropertiesFile() throws IOException
+ {
+ Properties props = new Properties();
+ String resourcePath = "/testTopology.properties";
+ InputStream is = this.getClass().getResourceAsStream(resourcePath);
+ if (is == null) {
+ fail("Could not load " + resourcePath);
+ }
+ props.load(is);
+ LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false))
+ .addFromProperties(props, null);
+
+ LogicalPlan dag = new LogicalPlan();
+ pb.populateDAG(dag);
+ dag.validate();
+
+ assertEquals("number of operator confs", 5, dag.getAllOperators().size());
+ assertEquals("number of root operators", 1, dag.getRootOperators().size());
+
+ StreamMeta s1 = dag.getStream("n1n2");
+ assertNotNull(s1);
+ assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
+
+ OperatorMeta operator3 = dag.getOperatorMeta("operator3");
+ assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
+
+ GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
+ assertEquals("myStringProperty " + doperator3, "myStringPropertyValueFromTemplate", doperator3.getMyStringProperty());
+ assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
+
+ OperatorMeta operator4 = dag.getOperatorMeta("operator4");
+ GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
+ assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
+ assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
+ assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
+
+ StreamMeta input1 = dag.getStream("inputStream");
+ assertNotNull(input1);
+ Assert.assertEquals("input1 source", dag.getOperatorMeta("inputOperator"), input1.getSource().getOperatorMeta());
+ Set<OperatorMeta> targetNodes = Sets.newHashSet();
+ for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
+ targetNodes.add(targetPort.getOperatorWrapper());
+ }
+
+ Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
+
+ }
+
+ @Test
+ public void testLoadFromJson() throws Exception
+ {
+ String resourcePath = "/testTopology.json";
+ InputStream is = this.getClass().getResourceAsStream(resourcePath);
+ if (is == null) {
+ fail("Could not load " + resourcePath);
+ }
+ StringWriter writer = new StringWriter();
+
+ IOUtils.copy(is, writer);
+ JSONObject json = new JSONObject(writer.toString());
+
+ Configuration conf = new Configuration(false);
+ conf.set(StreamingApplication.DT_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");
+
+ LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
+ dag.validate();
+
+ assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
+ Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
+ stringCodecsMap.put(Integer.class, Integer2String.class);
+ assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
+ assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());
+
+ assertEquals("number of operator confs", 5, dag.getAllOperators().size());
+ assertEquals("number of root operators", 1, dag.getRootOperators().size());
+
+ StreamMeta s1 = dag.getStream("n1n2");
+ assertNotNull(s1);
+ assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
+
+ OperatorMeta input = dag.getOperatorMeta("inputOperator");
+ TestStatsListener tsl = new TestStatsListener();
+ tsl.setIntProp(222);
+ List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
+ assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
+ for(OutputPortMeta opm : input.getOutputStreams().keySet()){
+ assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
+ }
+
+ OperatorMeta operator3 = dag.getOperatorMeta("operator3");
+ assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
+
+ GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
+ assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
+ assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
+
+ OperatorMeta operator4 = dag.getOperatorMeta("operator4");
+ GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
+ assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
+ assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
+ assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
+
+ StreamMeta input1 = dag.getStream("inputStream");
+ assertNotNull(input1);
+ OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
+ Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
+ Set<OperatorMeta> targetNodes = Sets.newHashSet();
+ for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
+ targetNodes.add(targetPort.getOperatorWrapper());
+ }
+ Assert.assertEquals("operator attribute " + inputOperator, 64, (int)inputOperator.getValue(OperatorContext.MEMORY_MB));
+ Assert.assertEquals("port attribute " + inputOperator, 8, (int)input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
+ Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
+ }
+
+ @Test
+ @SuppressWarnings("UnnecessaryBoxing")
+ public void testAppLevelAttributes()
+ {
+ String appName = "app1";
+
+ Properties props = new Properties();
+ props.put(StreamingApplication.DT_PREFIX + DAG.MASTER_MEMORY_MB.getName(), "123");
+ props.put(StreamingApplication.DT_PREFIX + DAG.CONTAINER_JVM_OPTIONS.getName(), "-Dlog4j.properties=custom_log4j.properties");
+ props.put(StreamingApplication.DT_PREFIX + DAG.APPLICATION_PATH.getName(), "/defaultdir");
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + "." + DAG.APPLICATION_PATH.getName(), "/otherdir");
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + "." + DAG.STREAMING_WINDOW_SIZE_MILLIS.getName(), "1000");
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ LogicalPlan dag = new LogicalPlan();
+
+ dagBuilder.populateDAG(dag);
+
+ dagBuilder.setApplicationConfiguration(dag, appName, null);
+
+ Assert.assertEquals("", "/otherdir", dag.getValue(DAG.APPLICATION_PATH));
+ Assert.assertEquals("", Integer.valueOf(123), dag.getValue(DAG.MASTER_MEMORY_MB));
+ Assert.assertEquals("", Integer.valueOf(1000), dag.getValue(DAG.STREAMING_WINDOW_SIZE_MILLIS));
+ Assert.assertEquals("", "-Dlog4j.properties=custom_log4j.properties", dag.getValue(DAG.CONTAINER_JVM_OPTIONS));
+
+ }
+ @Test
+ @SuppressWarnings("UnnecessaryBoxing")
+ public void testAppLevelProperties() {
+ String appName ="app1";
+ Properties props =new Properties();
+ props.put(StreamingApplication.DT_PREFIX + "application."+appName+".testprop1","10");
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".prop.testprop2", "100");
+ props.put(StreamingApplication.DT_PREFIX + "application.*.prop.testprop3","1000");
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".inncls.a", "10000");
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ LogicalPlan dag = new LogicalPlan();
+ TestApplication app1Test=new TestApplication();
+
+ dagBuilder.setApplicationConfiguration(dag, appName,app1Test);
+ Assert.assertEquals("",Integer.valueOf(10),app1Test.getTestprop1());
+ Assert.assertEquals("",Integer.valueOf(100),app1Test.getTestprop2());
+ Assert.assertEquals("",Integer.valueOf(1000),app1Test.getTestprop3());
+ Assert.assertEquals("",Integer.valueOf(10000),app1Test.getInncls().getA());
+ }
+
+ @Test
+ public void testPrepareDAG() {
+ final MutableBoolean appInitialized = new MutableBoolean(false);
+ StreamingApplication app = new StreamingApplication() {
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ Assert.assertEquals("", "hostname:9090", dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
+ dag.setAttribute(DAG.GATEWAY_CONNECT_ADDRESS, "hostname:9091");
+ appInitialized.setValue(true);
+ }
+ };
+ Configuration conf = new Configuration(false);
+ conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
+ LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
+
+ LogicalPlan dag = new LogicalPlan();
+ pb.prepareDAG(dag, app, "testconfig");
+
+ Assert.assertTrue("populateDAG called", appInitialized.booleanValue());
+ Assert.assertEquals("populateDAG overrides attribute", "hostname:9091", dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
+ }
+
+ @Test
+ public void testOperatorConfigurationLookup() {
+
+ Properties props = new Properties();
+
+ // match operator by name
+ props.put(StreamingApplication.DT_PREFIX + "template.matchId1.matchIdRegExp", ".*operator1.*");
+ props.put(StreamingApplication.DT_PREFIX + "template.matchId1.stringProperty2", "stringProperty2Value-matchId1");
+ props.put(StreamingApplication.DT_PREFIX + "template.matchId1.nested.property", "nested.propertyValue-matchId1");
+
+ // match class name, lower priority
+ props.put(StreamingApplication.DT_PREFIX + "template.matchClass1.matchClassNameRegExp", ".*" + ValidationTestOperator.class.getSimpleName());
+ props.put(StreamingApplication.DT_PREFIX + "template.matchClass1.stringProperty2", "stringProperty2Value-matchClass1");
+
+ // match class name
+ props.put(StreamingApplication.DT_PREFIX + "template.t2.matchClassNameRegExp", ".*"+GenericTestOperator.class.getSimpleName());
+ props.put(StreamingApplication.DT_PREFIX + "template.t2.myStringProperty", "myStringPropertyValue");
+
+ // direct setting
+ props.put(StreamingApplication.DT_PREFIX + "operator.operator3.emitFormat", "emitFormatValue");
+
+ LogicalPlan dag = new LogicalPlan();
+ Operator operator1 = dag.addOperator("operator1", new ValidationTestOperator());
+ Operator operator2 = dag.addOperator("operator2", new ValidationTestOperator());
+ Operator operator3 = dag.addOperator("operator3", new GenericTestOperator());
+
+ LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false));
+ LOG.debug("calling addFromProperties");
+ pb.addFromProperties(props, null);
+
+ Map<String, String> configProps = pb.getProperties(dag.getMeta(operator1), "appName");
+ Assert.assertEquals("" + configProps, 2, configProps.size());
+ Assert.assertEquals("" + configProps, "stringProperty2Value-matchId1", configProps.get("stringProperty2"));
+ Assert.assertEquals("" + configProps, "nested.propertyValue-matchId1", configProps.get("nested.property"));
+
+ configProps = pb.getProperties(dag.getMeta(operator2), "appName");
+ Assert.assertEquals("" + configProps, 1, configProps.size());
+ Assert.assertEquals("" + configProps, "stringProperty2Value-matchClass1", configProps.get("stringProperty2"));
+
+ configProps = pb.getProperties(dag.getMeta(operator3), "appName");
+ Assert.assertEquals("" + configProps, 2, configProps.size());
+ Assert.assertEquals("" + configProps, "myStringPropertyValue", configProps.get("myStringProperty"));
+ Assert.assertEquals("" + configProps, "emitFormatValue", configProps.get("emitFormat"));
+
+ }
+
+ @Test
+ public void testSetOperatorProperties() {
+
+ Configuration conf = new Configuration(false);
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.myStringProperty", "myStringPropertyValue");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.stringArrayField", "a,b,c");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty.key1", "key1Val");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key1.dot)", "key1dotVal");
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.prop.mapProperty(key2.dot)", "key2dotVal");
+
+ LogicalPlan dag = new LogicalPlan();
+ GenericTestOperator o1 = dag.addOperator("o1", new GenericTestOperator());
+ ValidationTestOperator o2 = dag.addOperator("o2", new ValidationTestOperator());
+
+ LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
+
+ pb.setOperatorProperties(dag, "testSetOperatorProperties");
+ Assert.assertEquals("o1.myStringProperty", "myStringPropertyValue", o1.getMyStringProperty());
+ Assert.assertArrayEquals("o2.stringArrayField", new String[] {"a", "b", "c"}, o2.getStringArrayField());
+
+ Assert.assertEquals("o2.mapProperty.key1", "key1Val", o2.getMapProperty().get("key1"));
+ Assert.assertEquals("o2.mapProperty(key1.dot)", "key1dotVal", o2.getMapProperty().get("key1.dot"));
+ Assert.assertEquals("o2.mapProperty(key2.dot)", "key2dotVal", o2.getMapProperty().get("key2.dot"));
+
+ }
+
+ @ApplicationAnnotation(name="AnnotatedAlias")
+ class AnnotatedApplication implements StreamingApplication {
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ }
+
+ }
+
+ @Test
+ public void testAppNameAttribute() {
+ StreamingApplication app = new AnnotatedApplication();
+ Configuration conf = new Configuration(false);
+ conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
+
+ LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
+
+ Properties properties = new Properties();
+ properties.put(StreamingApplication.DT_PREFIX + "application.TestAliasApp.class", app.getClass().getName());
+
+ builder.addFromProperties(properties, null);
+
+ LogicalPlan dag = new LogicalPlan();
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+ dag.setAttribute(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME, "testApp");
+ builder.prepareDAG(dag, app, appPath);
+
+ Assert.assertEquals("Application name", "testApp", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
+ }
+
+ @Test
+ public void testAppAlias() {
+ StreamingApplication app = new AnnotatedApplication();
+ Configuration conf = new Configuration(false);
+ conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
+
+ LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
+
+ Properties properties = new Properties();
+ properties.put(StreamingApplication.DT_PREFIX + "application.TestAliasApp.class", app.getClass().getName());
+
+ builder.addFromProperties(properties, null);
+
+ LogicalPlan dag = new LogicalPlan();
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+ builder.prepareDAG(dag, app, appPath);
+
+ Assert.assertEquals("Application name", "TestAliasApp", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
+ }
+
+
+ @Test
+ public void testAppAnnotationAlias() {
+ StreamingApplication app = new AnnotatedApplication();
+ Configuration conf = new Configuration(false);
+ conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
+
+ LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);
+
+ LogicalPlan dag = new LogicalPlan();
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+ builder.prepareDAG(dag, app, appPath);
+
+ Assert.assertEquals("Application name", "AnnotatedAlias", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
+ }
+
+ @Test
+ @SuppressWarnings( {"UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes"})
+ public void testOperatorLevelAttributes() {
+ String appName = "app1";
+ StreamingApplication app = new StreamingApplication() {
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ dag.addOperator("operator1", GenericTestOperator.class);
+ dag.addOperator("operator2", GenericTestOperator.class);
+ }
+ };
+
+ Properties props = new Properties();
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
+ props.put(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "2");
+ props.put(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.STATS_LISTENERS.getName(), PartitionLoadWatch.class.getName());
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "20");
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ Assert.assertEquals("", Integer.valueOf(20), dag.getOperatorMeta("operator1").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
+ Assert.assertEquals("", Integer.valueOf(2), dag.getOperatorMeta("operator2").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
+ Assert.assertEquals("", PartitionLoadWatch.class, dag.getOperatorMeta("operator2").getValue(OperatorContext.STATS_LISTENERS).toArray()[0].getClass());
+ }
+
+ @Test
+ public void testOperatorLevelProperties() {
+ String appName = "app1";
+ final GenericTestOperator operator1 = new GenericTestOperator();
+ final GenericTestOperator operator2 = new GenericTestOperator();
+ StreamingApplication app = new StreamingApplication() {
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ dag.addOperator("operator1", operator1);
+ dag.addOperator("operator2", operator2);
+ }
+ };
+
+ Properties props = new Properties();
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
+ props.put(StreamingApplication.DT_PREFIX + "operator.*.myStringProperty", "pv1");
+ props.put(StreamingApplication.DT_PREFIX + "operator.*.booleanProperty", Boolean.TRUE.toString());
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1.myStringProperty", "apv1");
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ Assert.assertEquals("apv1", operator1.getMyStringProperty());
+ Assert.assertEquals("pv1", operator2.getMyStringProperty());
+ Assert.assertEquals(true, operator2.isBooleanProperty());
+ }
+
+ @Test
+ public void testApplicationLevelParameter()
+ {
+ String appName = "app1";
+ final GenericTestOperator operator1 = new GenericTestOperator();
+ final GenericTestOperator operator2 = new GenericTestOperator();
+ StreamingApplication app = new StreamingApplication()
+ {
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ dag.addOperator("operator1", operator1);
+ dag.addOperator("operator2", operator2);
+ }
+ };
+
+ Properties props = new Properties();
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
+ props.put(StreamingApplication.DT_PREFIX + "operator.*.myStringProperty", "foo ${xyz} bar ${zzz} baz");
+ props.put(StreamingApplication.DT_PREFIX + "operator.*.booleanProperty", Boolean.TRUE.toString());
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1.myStringProperty", "apv1");
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+
+ Configuration vars = new Configuration(false);
+ vars.set("xyz", "123");
+ vars.set("zzz", "456");
+ dagBuilder.addFromProperties(props, vars);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ Assert.assertEquals("apv1", operator1.getMyStringProperty());
+ Assert.assertEquals("foo 123 bar 456 baz", operator2.getMyStringProperty());
+ Assert.assertEquals(true, operator2.isBooleanProperty());
+ }
+
+ @Test
+ @SuppressWarnings("UnnecessaryBoxing")
+ public void testPortLevelAttributes() {
+ String appName = "app1";
+ SimpleTestApplication app = new SimpleTestApplication();
+
+ Properties props = new Properties();
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".class", app.getClass().getName());
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator1.port.*." + PortContext.QUEUE_CAPACITY.getName(), "" + 16 * 1024);
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator2.inputport.inport1." + PortContext.QUEUE_CAPACITY.getName(), "" + 32 * 1024);
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator2.outputport.outport1." + PortContext.QUEUE_CAPACITY.getName(), "" + 32 * 1024);
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator3.port.*." + PortContext.QUEUE_CAPACITY.getName(), "" + 16 * 1024);
+ props.put(StreamingApplication.DT_PREFIX + "application." + appName + ".operator.operator3.inputport.inport2." + PortContext.QUEUE_CAPACITY.getName(), "" + 32 * 1024);
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ OperatorMeta om1 = dag.getOperatorMeta("operator1");
+ Assert.assertEquals("", Integer.valueOf(16 * 1024), om1.getMeta(app.gt1.outport1).getValue(PortContext.QUEUE_CAPACITY));
+ OperatorMeta om2 = dag.getOperatorMeta("operator2");
+ Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(app.gt2.inport1).getValue(PortContext.QUEUE_CAPACITY));
+ Assert.assertEquals("", Integer.valueOf(32 * 1024), om2.getMeta(app.gt2.outport1).getValue(PortContext.QUEUE_CAPACITY));
+ OperatorMeta om3 = dag.getOperatorMeta("operator3");
+ Assert.assertEquals("", Integer.valueOf(16 * 1024), om3.getMeta(app.gt3.inport1).getValue(PortContext.QUEUE_CAPACITY));
+ Assert.assertEquals("", Integer.valueOf(32 * 1024), om3.getMeta(app.gt3.inport2).getValue(PortContext.QUEUE_CAPACITY));
+ }
+
+
+ @Test
+ public void testInvalidAttribute() throws Exception {
+ Assert.assertNotSame(0, com.datatorrent.api.Context.DAGContext.serialVersionUID);
+ Attribute<String> attribute = new Attribute<>("", null);
+
+ Field nameField = Attribute.class.getDeclaredField("name");
+ nameField.setAccessible(true);
+ nameField.set(attribute, "NOT_CONFIGURABLE");
+ nameField.setAccessible(false);
+
+ ContextUtils.addAttribute(com.datatorrent.api.Context.DAGContext.class, attribute);
+ AttributeParseUtils.initialize();
+ ConfElement.initialize();
+
+ // attribute that cannot be configured
+
+ Properties props = new Properties();
+ props.put(StreamingApplication.DT_PREFIX + "attr.NOT_CONFIGURABLE", "value");
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ try {
+ dagBuilder.prepareDAG(new LogicalPlan(), null, "");
+ Assert.fail("Exception expected");
+ } catch (Exception e) {
+ Assert.assertThat("Attribute not configurable", e.getMessage(), RegexMatcher.matches("Attribute does not support property configuration: NOT_CONFIGURABLE.*"));
+ }
+
+ ContextUtils.removeAttribute(com.datatorrent.api.Context.DAGContext.class, attribute);
+ AttributeParseUtils.initialize();
+ ConfElement.initialize();
+
+ // invalid attribute name
+ props = new Properties();
+ String invalidAttribute = StreamingApplication.DT_PREFIX + "attr.INVALID_NAME";
+ props.put(invalidAttribute, "value");
+
+ try {
+ new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
+ Assert.fail("Exception expected");
+ } catch (Exception e) {
+ LOG.debug("Exception message: {}", e);
+ Assert.assertThat("Invalid attribute name", e.getMessage(), RegexMatcher.matches("Invalid attribute reference: " + invalidAttribute));
+ }
+ }
+
+ @Test
+ public void testAttributesCodec() {
+ Assert.assertNotSame(null, new Long[] {com.datatorrent.api.Context.DAGContext.serialVersionUID, OperatorContext.serialVersionUID, PortContext.serialVersionUID});
+ @SuppressWarnings("unchecked")
+ Set<Class<? extends Context>> contextClasses = Sets.newHashSet(com.datatorrent.api.Context.DAGContext.class, OperatorContext.class, PortContext.class);
+ for (Class<?> c : contextClasses) {
+ for (Attribute<Object> attr : AttributeInitializer.getAttributes(c)) {
+ Assert.assertNotNull(attr.name + " codec", attr.codec);
+ }
+ }
+ }
+
+ @Test
+ public void testTupleClassAttr() throws Exception
+ {
+ String resourcePath = "/schemaTestTopology.json";
+ InputStream is = this.getClass().getResourceAsStream(resourcePath);
+ if (is == null) {
+ fail("Could not load " + resourcePath);
+ }
+ StringWriter writer = new StringWriter();
+
+ IOUtils.copy(is, writer);
+ JSONObject json = new JSONObject(writer.toString());
+
+ Configuration conf = new Configuration(false);
+
+ LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
+ dag.validate();
+
+ OperatorMeta operator1 = dag.getOperatorMeta("operator1");
+ assertEquals("operator1.classname", SchemaTestOperator.class, operator1.getOperator().getClass());
+
+ StreamMeta input1 = dag.getStream("inputStream");
+ assertNotNull(input1);
+ for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
+ Assert.assertEquals("tuple class name required", TestSchema.class, targetPort.getAttributes().get(PortContext.TUPLE_CLASS));
+ }
+ }
+
+ @Test(expected = ValidationException.class)
+ public void testTupleClassAttrValidation() throws Exception
+ {
+ String resourcePath = "/schemaTestTopology.json";
+ InputStream is = this.getClass().getResourceAsStream(resourcePath);
+ if (is == null) {
+ fail("Could not load " + resourcePath);
+ }
+ StringWriter writer = new StringWriter();
+
+ IOUtils.copy(is, writer);
+ JSONObject json = new JSONObject(writer.toString());
+
+ //removing schema so that validation fails
+ json.getJSONArray("streams").getJSONObject(0).remove("schema");
+ Configuration conf = new Configuration(false);
+
+ LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
+ LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
+
+ dag.validate();
+ }
+
+ @Test
+ public void testTestTupleClassAttrSetFromConfig()
+ {
+ Configuration conf = new Configuration(false);
+ conf.set(StreamingApplication.DT_PREFIX + "operator.o2.port.schemaRequiredPort.attr.TUPLE_CLASS",
+ "com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest$TestSchema");
+
+ StreamingApplication streamingApplication = new StreamingApplication()
+ {
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ TestGeneratorInputOperator o1 = dag.addOperator("o1", new TestGeneratorInputOperator());
+ SchemaTestOperator o2 = dag.addOperator("o2", new SchemaTestOperator());
+ dag.addStream("stream", o1.outport, o2.schemaRequiredPort);
+ }
+ };
+ LogicalPlan dag = new LogicalPlan();
+ LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
+ lpc.prepareDAG(dag, streamingApplication, "app");
+ dag.validate();
+ }
+
+ /*
+ * This test and all of the following ambiguous attribute tests verify that when an ambiguous attribute
+ * name is provided, all the corresponding attributes are set.
+ * <br/><br/>
+ * <b>Note:</b> Ambiguous attribute means that when multiple attributes with the same
+ * simple name exist for multiple types of dag elements (like operators and ports).
+ * An example of such attributes are the com.datatorrent.api.Context.OperatorContext.AUTO_RECORD
+ * and com.datatorrent.api.Context.PortContext.AUTO_RECORD.
+ * <br/><br/>
+ * This test should set the attribute on the operators and ports.
+ */
+ /**
+ * This test checks if an ambiguous DAG attribute does not get set on operators.
+ */
+ @Test
+ public void testDagAttributeNotSetOnOperator()
+ {
+ dagOperatorAttributeHelper(true);
+ }
+
+ @Test
+ public void testAmbiguousAttributeSetOnOperatorAndNotDAG()
+ {
+ dagOperatorAttributeHelper(false);
+ }
+
+ private void dagOperatorAttributeHelper(boolean attrOnDag)
+ {
+ String attributeName = null;
+
+ if (attrOnDag) {
+ attributeName = DAGContext.CHECKPOINT_WINDOW_COUNT.getSimpleName();
+ }
+ else {
+ attributeName = OperatorContext.class.getCanonicalName() + LogicalPlanConfiguration.KEY_SEPARATOR + DAGContext.CHECKPOINT_WINDOW_COUNT.getSimpleName();
+ }
+
+ Properties props = new Properties();
+ String propName = StreamingApplication.DT_PREFIX + StramElement.ATTR.getValue() + LogicalPlanConfiguration.KEY_SEPARATOR + attributeName;
+ props.put(propName, "5");
+
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ OperatorMeta om1 = dag.getOperatorMeta("operator1");
+
+ if (attrOnDag) {
+ Assert.assertNotEquals((Integer)5, om1.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT));
+ } else {
+ Assert.assertEquals((Integer)5, om1.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT));
+ }
+ }
+
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testRootLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX, null, Boolean.TRUE, true, true);
+ }
+
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testApplicationLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX + "application" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, null, Boolean.TRUE, true, true);
+ }
+
+ /**
+ * This should only set the attribute on the operator
+ */
+ @Test
+ public void testOperatorLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX + "operator" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, null, Boolean.TRUE, true, false);
+ }
+
+ /**
+ * This should only set the attribute on the port
+ */
+ @Test
+ public void testPortLevelAmbiguousAttributeSimple()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX + "port" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, null, Boolean.TRUE, false, true);
+ }
+
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testRootLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD, StreamingApplication.DT_PREFIX,
+ PortContext.class.getCanonicalName(), Boolean.TRUE, false, true);
+ }
+
+ /**
+ * This test should set the attribute on the operators and ports.
+ */
+ @Test
+ public void testApplicationLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX + "application" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, PortContext.class.getCanonicalName(),
+ Boolean.TRUE, false, true);
+ }
+
+ /**
+ * This should only set the attribute on the operator
+ */
+ @Test
+ public void testOperatorLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX + "operator" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, OperatorContext.class.getCanonicalName(),
+ Boolean.TRUE, true, false);
+ }
+
+ /**
+ * This should only set the attribute on the port
+ */
+ @Test
+ public void testOperatorLevelAmbiguousAttributeComplex2()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX + "operator" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, PortContext.class.getCanonicalName(),
+ Boolean.TRUE, false, true);
+ }
+
+ /**
+ * This should only set the attribute on the port
+ */
+ @Test
+ public void testPortLevelAmbiguousAttributeComplex()
+ {
+ testAttributeAmbiguousSimpleHelper(Context.OperatorContext.AUTO_RECORD, Context.PortContext.AUTO_RECORD,
+ StreamingApplication.DT_PREFIX + "port" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, PortContext.class.getCanonicalName(),
+ Boolean.TRUE, false, true);
+ }
+
+ private void testAttributeAmbiguousSimpleHelper(Attribute<?> attributeObjOperator, Attribute<?> attributeObjPort,
+ String root, String contextClass, Object val, boolean operatorSet,
+ boolean portSet)
+ {
+ Properties props = propertiesBuilder(attributeObjOperator.getSimpleName(), root, contextClass, val);
+
+ simpleAttributeOperatorHelperAssert(attributeObjOperator, props, val, operatorSet);
+
+ simpleNamePortAssertHelperAssert(attributeObjPort, props, val, portSet);
+ }
+
+ @Test
+ public void testRootLevelAttributeSimpleNameOperator()
+ {
+ simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB, StreamingApplication.DT_PREFIX, true, 4096, true, true);
+ }
+
+ @Test
+ public void testRootLevelStorageAgentSimpleNameOperator()
+ {
+ MockStorageAgent mockAgent = new MockStorageAgent();
+
+ simpleAttributeOperatorHelper(OperatorContext.STORAGE_AGENT, StreamingApplication.DT_PREFIX, true, mockAgent, true, false);
+ }
+
+ @Test
+ public void testRootLevelAttributeSimpleNameOperatorNoScope()
+ {
+ simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB, StreamingApplication.DT_PREFIX, true, 4096, true, false);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNameOperator()
+ {
+ simpleAttributeOperatorHelper(OperatorContext.MEMORY_MB, StreamingApplication.DT_PREFIX + "application" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "SimpleTestApp" + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true, 4096, true, true);
+ }
+
+ private void simpleAttributeOperatorHelper(Attribute<?> attributeObj, String root, boolean simpleName,
+ Object val, boolean set, boolean scope)
+ {
+ Properties props = propertiesBuilderOperator(attributeObj.getSimpleName(), root, simpleName,
+ val, scope);
+
+ simpleAttributeOperatorHelperAssert(attributeObj, props, val, set);
+ }
+
+ private void simpleAttributeOperatorHelperAssert(Attribute<?> attributeObj, Properties props, Object val, boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ OperatorMeta om1 = dag.getOperatorMeta("operator1");
+
+ if (set) {
+ Assert.assertEquals(val, om1.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(val, om1.getValue(attributeObj));
+ }
+
+ OperatorMeta om2 = dag.getOperatorMeta("operator2");
+
+ if (set) {
+ Assert.assertEquals(val, om2.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(val, om2.getValue(attributeObj));
+ }
+
+ OperatorMeta om3 = dag.getOperatorMeta("operator3");
+
+ if (set) {
+ Assert.assertEquals(val, om3.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(val, om3.getValue(attributeObj));
+ }
+ }
+
+ /* Port tests */
+ @Test
+ public void testRootLevelAttributeSimpleNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX,
+ true, (Integer)4096, true, true);
+ }
+
+ @Test
+ public void testRootLevelAttributeSimpleNamePortNoScope()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX,
+ true, (Integer)4096, true, false);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeSimpleNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "operator" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true, (Integer)4096, true, true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "application" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "SimpleTestApp" + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true, (Integer)4096, true, true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX, false,
+ (Integer)4096, true, true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNamePortNoScope()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX, false,
+ (Integer)4096, true, false);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeComplexNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "operator" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false, (Integer)4096, true, true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeComplexNamePort()
+ {
+ simpleAttributePortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "application" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "SimpleTestApp" + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false, (Integer)4096, true, true);
+ }
+
+ /* Input port tests */
+ @Test
+ public void testRootLevelAttributeSimpleNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX, true,
+ (Integer)4096, true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeSimpleNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "operator" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, true, (Integer)4096, true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "application" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "SimpleTestApp" + LogicalPlanConfiguration.KEY_SEPARATOR,
+ true, (Integer)4096, true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX, false, (Integer)4096, true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeComplexNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "operator" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR, false,
+ (Integer)4096, true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeComplexNameInputPort()
+ {
+ simpleAttributeInputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "application" +
+ LogicalPlanConfiguration.KEY_SEPARATOR + "SimpleTestApp" + LogicalPlanConfiguration.KEY_SEPARATOR,
+ false, (Integer)4096, true);
+ }
+
+ /* Output port tests */
+ @Test
+ public void testRootLevelAttributeSimpleNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX, true, (Integer)4096, true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeSimpleNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "operator" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, true, (Integer)4096, true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeSimpleNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "application" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "SimpleTestApp" + LogicalPlanConfiguration.KEY_SEPARATOR, true, (Integer)4096, true);
+ }
+
+ @Test
+ public void testRootLevelAttributeComplexNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX, false, (Integer)4096, true);
+ }
+
+ @Test
+ public void testOperatorLevelAttributeComplexNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "operator" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "*" + LogicalPlanConfiguration.KEY_SEPARATOR, false, (Integer)4096, true);
+ }
+
+ @Test
+ public void testApplicationLevelAttributeComplexNameOutputPort()
+ {
+ simpleAttributeOutputPortHelper(PortContext.QUEUE_CAPACITY, StreamingApplication.DT_PREFIX + "application" + LogicalPlanConfiguration.KEY_SEPARATOR +
+ "SimpleTestApp" + LogicalPlanConfiguration.KEY_SEPARATOR, false, (Integer)4096, true);
+ }
+
+ /* Helpers for building ports */
+ private void simpleAttributePortHelper(Attribute<?> attributeObj, String root, boolean simpleName, Object val, boolean set, boolean scope)
+ {
+ Properties props = propertiesBuilderPort(attributeObj.getSimpleName(), root, simpleName, val, scope);
+
+ simpleNamePortAssertHelperAssert(attributeObj, props, val, set);
+ }
+
+ private void simpleAttributeInputPortHelper(Attribute<?> attributeObj, String root, boolean simpleName, Object val, boolean set)
+ {
+ Properties props = propertiesBuilderInputPort(attributeObj.getSimpleName(), root, simpleName, val);
+
+ simpleNameInputPortAssertHelperAssert(attributeObj, props, val, set);
+ simpleNameOutputPortAssertHelperAssert(attributeObj, props, val, !set);
+ }
+
+ private void simpleAttributeOutputPortHelper(Attribute<?> attributeObj, String root, boolean simpleName, Object val, boolean set)
+ {
+ Properties props = propertiesBuilderOutputPort(attributeObj.getSimpleName(), root, simpleName, val);
+
+ simpleNameOutputPortAssertHelperAssert(attributeObj, props, val, set);
+ simpleNameInputPortAssertHelperAssert(attributeObj, props, val, !set);
+ }
+
+ private void simpleNamePortAssertHelperAssert(Attribute<?> attributeObj, Properties props, Object val, boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ simpleNamePortAssertHelper(attributeObj, dag, "operator1", val, set);
+ simpleNamePortAssertHelper(attributeObj, dag, "operator2", val, set);
+ simpleNamePortAssertHelper(attributeObj, dag, "operator3", val, set);
+ }
+
+ private void simpleNameInputPortAssertHelperAssert(Attribute<?> attributeObj, Properties props, Object val, boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ simpleNameInputPortAssertHelper(attributeObj, dag, "operator1", val, set);
+ simpleNameInputPortAssertHelper(attributeObj, dag, "operator2", val, set);
+ simpleNameInputPortAssertHelper(attributeObj, dag, "operator3", val, set);
+ }
+
+ private void simpleNameOutputPortAssertHelperAssert(Attribute<?> attributeObj, Properties props, Object val, boolean set)
+ {
+ SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();
+
+ LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
+ dagBuilder.addFromProperties(props, null);
+
+ String appPath = app.getClass().getName().replace(".", "/") + ".class";
+
+ LogicalPlan dag = new LogicalPlan();
+ dagBuilder.prepareDAG(dag, app, appPath);
+
+ simpleNameOutputPortAssertHelper(attributeObj, dag, "operator1", val, set);
+ simpleNameOutputPortAssertHelper(attributeObj, dag, "operator2", val, set);
+ simpleNameOutputPortAssertHelper(attributeObj, dag, "operator3", val, set);
+ }
+
+ private void simpleNamePortAssertHelper(Attribute<?> attributeObj, LogicalPlan dag, String operatorName, Object queueCapacity, boolean set)
+ {
+ simpleNameInputPortAssertHelper(attributeObj, dag, operatorName, queueCapacity, set);
+ simpleNameOutputPortAssertHelper(attributeObj, dag, operatorName, queueCapacity, set);
+ }
+
+ private void simpleNameInputPortAssertHelper(Attribute<?> attributeObj, LogicalPlan dag, String operatorName, Object queueCapacity, boolean set)
+ {
+ OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
+
+ for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
+ if (set) {
+ Assert.assertEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
+ }
+ }
+ }
+
+ private void simpleNameOutputPortAssertHelper(Attribute<?> attributeObj, LogicalPlan dag, String operatorName, Object queueCapacity, boolean set)
+ {
+ OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
+
+ for (OutputPortMeta outputPortMeta: operatorMeta.getOutputStreams().keySet()) {
+ if (set) {
+ Assert.assertEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
+ } else {
+ Assert.assertNotEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
+ }
+ }
+ }
+
+ /* Helpers for building properties */
+ private Properties propertiesBuilder(String attributeName, String root, String contextClass, Object val)
+ {
+ boolean simpleName = contextClass == null;
+
+ if (!simpleName) {
+ attributeName = contextClass + LogicalPlanConfiguration.KEY_SEPARATOR + attributeName;
+ }
+
+ Properties props = new Properties();
+ String propName = root + StramElement.ATTR.getValue() + LogicalPlanConfiguration.KEY_SEPARATOR + attributeName;
+ props.put(propName, val.toString());
+
+ return props;
+ }
+
+ private Properties propertiesBuilderOperator(String attributeName, String root, boolean simpleName, Object val, boolean addOperator)
+ {
+ String contextClass = simpleName ? null : OperatorContext.class.getCanonicalName();
+
+ if (addOperator) {
+ root += "operator" + LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR;
+ }
+
+ return propertiesBuilder(attributeName, root, contextClass, val);
+ }
+
+ private Properties propertiesBuilderPort(String attributeName, String root, boolean simpleName, Object val, boolean addPort)
+ {
+ String contextClass = simpleName ? null : PortContext.class.getCanonicalName();
+
+ if (addPort) {
+ root += "port" + LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR;
+ }
+
+ return propertiesBuilder(attributeName, root, contextClass, val);
+ }
+
+ private Properties propertiesBuilderInputPort(String attributeName, String root, boolean simpleName, Object val)
+ {
+ String contextClass = simpleName ? null: PortContext.class.getCanonicalName();
+
+ root += "inputport" + LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR;
+
+ return propertiesBuilder(attributeName, root, contextClass, val);
+ }
+
+ private Properties propertiesBuilderOutputPort(String attributeName, String root, boolean simpleName, Object val)
+ {
+ String contextClass = simpleName ? null: PortContext.class.getCanonicalName();
+
+ root += "outputport" + LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR;
+
+ return propertiesBuilder(attributeName, root, contextClass, val);
+ }
+
+ /**
+ * Note: If the same name is given to an Attribute specified in multiple Context classes, then the type of that
+ * Attribute is required to be the same accross all Context classes. This is required because if a simple attribute
+ * name is specified in a properties file at the top level context then that attribute needs to be set in all child configurations. If
+ * there were multiple Attributes specified in different Contexts with the same name, but a different type, then
+ * it would not be possible to set the values of Attributes specified by a simple attribute name in the root
+ * context of a properties file. If this were the case, then adding another Attribute with the same name as a pre-existing Attribute to a new Context
+ * class would be a backwards incompatible change.
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testErrorSameAttrMultipleTypes()
+ {
+ //Trigger initialization of attributes for existing Contexts.
+ LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
+
+ Exception ex = null;
+ try {
+ ContextUtils.buildAttributeMaps(Sets.newHashSet(MockContext1.class, MockContext2.class));
+ } catch (ValidationException e) {
+ ex = e;
+ }
+
+ Assert.assertNotNull(ex);
+ Assert.assertTrue(ex.getMessage().contains("is defined with two different types in two different context classes:"));
+
+ //Clear test data from Context.
+ ContextUtils.initialize();
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(LogicalPlanConfigurationTest.class);
+
+ public static class TestApplication implements StreamingApplication {
+ Integer testprop1;
+ Integer testprop2;
+ Integer testprop3;
+ TestInnerClass inncls;
+ public TestApplication() {
+ inncls=new TestInnerClass();
+ }
+
+ public Integer getTestprop1() {
+ return testprop1;
+ }
+
+ public void setTestprop1(Integer testprop1) {
+ this.testprop1 = testprop1;
+ }
+
+ public Integer getTestprop2() {
+ return testprop2;
+ }
+
+ public void setTestprop2(Integer testprop2) {
+ this.testprop2 = testprop2;
+ }
+
+ public Integer getTestprop3() {
+ return testprop3;
+ }
+
+ public void setTestprop3(Integer testprop3) {
+ this.testprop3 = testprop3;
+ }
+
+ public TestInnerClass getInncls() {
+ return inncls;
+ }
+
+ public void setInncls(TestInnerClass inncls) {
+ this.inncls = inncls;
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf) {
+
+ }
+ public class TestInnerClass{
+ Integer a;
+
+ public Integer getA() {
+ return a;
+ }
+
+ public void setA(Integer a) {
+ this.a = a;
+ }
+ }
+ }
+
+ public static class TestStatsListener implements StatsListener{
+
+ private int intProp;
+
+ public TestStatsListener()
+ {
+ }
+
+ @Override
+ public Response processStats(BatchedOperatorStats stats)
+ {
+ return null;
+ }
+
+ public int getIntProp()
+ {
+ return intProp;
+ }
+
+ public void setIntProp(int intProp)
+ {
+ this.intProp = intProp;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + intProp;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ TestStatsListener other = (TestStatsListener) obj;
+ if (intProp != other.intProp)
+ return false;
+ return true;
+ }
+ }
+
+ public static class TestSchema
+ {
+ }
+
+ public static class SimpleTestApplication implements StreamingApplication
+ {
+ public final GenericTestOperator gt1 = new GenericTestOperator();
+ public final GenericTestOperator gt2 = new GenericTestOperator();
+ public final GenericTestOperator gt3 = new GenericTestOperator();
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ dag.addOperator("operator1", gt1);
+ dag.addOperator("operator2", gt2);
+ dag.addOperator("operator3", gt3);
+ dag.addStream("s1", gt1.outport1, gt2.inport1);
+ dag.addStream("s2", gt2.outport1, gt3.inport1, gt3.inport2);
+ }
+ };
+
+ public static interface MockContext1 extends Context
+ {
+ /**
+ * Number of tuples the poll buffer can cache without blocking the input stream to the port.
+ */
+ Attribute<Integer> TEST_ATTR = new Attribute<>(1024);
+
+ @SuppressWarnings("FieldNameHidesFieldInSuperclass")
+ long serialVersionUID = AttributeMap.AttributeInitializer.initialize(MockContext1.class);
+ }
+
+ public static interface MockContext2 extends Context
+ {
+ /**
+ * Number of tuples the poll buffer can cache without blocking the input stream to the port.
+ */
+ Attribute<Boolean> TEST_ATTR = new Attribute<>(false);
+
+ @SuppressWarnings("FieldNameHidesFieldInSuperclass")
+ long serialVersionUID = AttributeMap.AttributeInitializer.initialize(MockContext2.class);
+ }
+
+ @ApplicationAnnotation(name="SimpleTestApp")
+ public static class SimpleTestApplicationWithName extends SimpleTestApplication
+ {
+ };
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogicalPlanConfigurationTest.class);
+}
+
[8/8] incubator-apex-core git commit: Merge branch
'APEX-28_pull_latest' into devel-3
Posted by cs...@apache.org.
Merge branch 'APEX-28_pull_latest' into devel-3
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/454fecca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/454fecca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/454fecca
Branch: refs/heads/devel-3
Commit: 454feccacd280996624db339e6270bb0f20ebccc
Parents: 7503dde 977093e
Author: Chandni Singh <cs...@apache.org>
Authored: Wed Sep 16 21:52:27 2015 -0700
Committer: Chandni Singh <cs...@apache.org>
Committed: Wed Sep 16 21:52:27 2015 -0700
----------------------------------------------------------------------
.../stram/plan/logical/LogicalPlan.java | 10 +-
.../plan/logical/LogicalPlanConfiguration.java | 1305 ++++++++++++---
.../plan/LogicalPlanConfigurationTest.java | 876 ----------
.../datatorrent/stram/plan/LogicalPlanTest.java | 988 ------------
.../logical/LogicalPlanConfigurationTest.java | 1511 ++++++++++++++++++
.../stram/plan/logical/LogicalPlanTest.java | 988 ++++++++++++
.../stram/plan/logical/MockStorageAgent.java | 67 +
.../src/test/resources/schemaTestTopology.json | 2 +-
engine/src/test/resources/testTopology.json | 4 +-
9 files changed, 3639 insertions(+), 2112 deletions(-)
----------------------------------------------------------------------
[3/8] incubator-apex-core git commit: APEX-28 #resolve
Posted by cs...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanTest.java b/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanTest.java
new file mode 100644
index 0000000..94dce6c
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/plan/logical/LogicalPlanTest.java
@@ -0,0 +1,988 @@
+/**
+ * Copyright (C) 2015 DataTorrent, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datatorrent.stram.plan.logical;
+
+import com.datatorrent.common.util.BaseOperator;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Serializable;
+import java.util.*;
+
+import javax.validation.*;
+import javax.validation.constraints.AssertTrue;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Pattern;
+
+import com.esotericsoftware.kryo.DefaultSerializer;
+import com.esotericsoftware.kryo.serializers.JavaSerializer;
+import com.google.common.collect.Maps;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+import com.datatorrent.api.*;
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.api.annotation.OperatorAnnotation;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.netlet.util.Slice;
+import com.datatorrent.stram.engine.GenericTestOperator;
+import com.datatorrent.stram.engine.TestGeneratorInputOperator;
+import com.datatorrent.stram.engine.TestNonOptionalOutportInputOperator;
+import com.datatorrent.stram.engine.TestOutputOperator;
+import com.datatorrent.stram.plan.logical.LogicalPlan;
+import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
+import com.datatorrent.stram.support.StramTestSupport.MemoryStorageAgent;
+import com.datatorrent.stram.support.StramTestSupport.RegexMatcher;
+
+public class LogicalPlanTest {
+
+ @Test
+ public void testCycleDetection() {
+ LogicalPlan dag = new LogicalPlan();
+
+ //NodeConf operator1 = b.getOrAddNode("operator1");
+ GenericTestOperator operator2 = dag.addOperator("operator2", GenericTestOperator.class);
+ GenericTestOperator operator3 = dag.addOperator("operator3", GenericTestOperator.class);
+ GenericTestOperator operator4 = dag.addOperator("operator4", GenericTestOperator.class);
+ //NodeConf operator5 = b.getOrAddNode("operator5");
+ //NodeConf operator6 = b.getOrAddNode("operator6");
+ GenericTestOperator operator7 = dag.addOperator("operator7", GenericTestOperator.class);
+
+ // strongly connect n2-n3-n4-n2
+ dag.addStream("n2n3", operator2.outport1, operator3.inport1);
+
+ dag.addStream("n3n4", operator3.outport1, operator4.inport1);
+
+ dag.addStream("n4n2", operator4.outport1, operator2.inport1);
+
+ // self referencing operator cycle
+ StreamMeta n7n7 = dag.addStream("n7n7", operator7.outport1, operator7.inport1);
+ try {
+ n7n7.addSink(operator7.inport1);
+ fail("cannot add to stream again");
+ } catch (Exception e) {
+ // expected, stream can have single input/output only
+ }
+
+ List<List<String>> cycles = new ArrayList<List<String>>();
+ dag.findStronglyConnected(dag.getMeta(operator7), cycles);
+ assertEquals("operator self reference", 1, cycles.size());
+ assertEquals("operator self reference", 1, cycles.get(0).size());
+ assertEquals("operator self reference", dag.getMeta(operator7).getName(), cycles.get(0).get(0));
+
+ // 3 operator cycle
+ cycles.clear();
+ dag.findStronglyConnected(dag.getMeta(operator4), cycles);
+ assertEquals("3 operator cycle", 1, cycles.size());
+ assertEquals("3 operator cycle", 3, cycles.get(0).size());
+ assertTrue("operator2", cycles.get(0).contains(dag.getMeta(operator2).getName()));
+ assertTrue("operator3", cycles.get(0).contains(dag.getMeta(operator3).getName()));
+ assertTrue("operator4", cycles.get(0).contains(dag.getMeta(operator4).getName()));
+
+ try {
+ dag.validate();
+ fail("validation should fail");
+ } catch (ValidationException e) {
+ // expected
+ }
+
+ }
+
+ public static class ValidationOperator extends BaseOperator {
+ public final transient DefaultOutputPort<Object> goodOutputPort = new DefaultOutputPort<Object>();
+
+ public final transient DefaultOutputPort<Object> badOutputPort = new DefaultOutputPort<Object>();
+ }
+
+ public static class CounterOperator extends BaseOperator {
+ final public transient InputPort<Object> countInputPort = new DefaultInputPort<Object>() {
+ @Override
+ final public void process(Object payload) {
+ }
+ };
+ }
+
+ @Test
+ public void testLogicalPlanSerialization() throws Exception {
+
+ LogicalPlan dag = new LogicalPlan();
+ dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
+
+ ValidationOperator validationNode = dag.addOperator("validationNode", ValidationOperator.class);
+ CounterOperator countGoodNode = dag.addOperator("countGoodNode", CounterOperator.class);
+ CounterOperator countBadNode = dag.addOperator("countBadNode", CounterOperator.class);
+ //ConsoleOutputOperator echoBadNode = dag.addOperator("echoBadNode", ConsoleOutputOperator.class);
+
+ // good tuples to counter operator
+ dag.addStream("goodTuplesStream", validationNode.goodOutputPort, countGoodNode.countInputPort);
+
+ // bad tuples to separate stream and echo operator
+ // (stream with 2 outputs)
+ dag.addStream("badTuplesStream", validationNode.badOutputPort, countBadNode.countInputPort);
+
+ Assert.assertEquals("number root operators", 1, dag.getRootOperators().size());
+ Assert.assertEquals("root operator id", "validationNode", dag.getRootOperators().get(0).getName());
+
+ dag.getContextAttributes(countGoodNode).put(OperatorContext.SPIN_MILLIS, 10);
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ LogicalPlan.write(dag, bos);
+
+ // System.out.println("serialized size: " + bos.toByteArray().length);
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+ LogicalPlan dagClone = LogicalPlan.read(bis);
+ Assert.assertNotNull(dagClone);
+ Assert.assertEquals("number operators in clone", dag.getAllOperators().size(), dagClone.getAllOperators().size());
+ Assert.assertEquals("number root operators in clone", 1, dagClone.getRootOperators().size());
+ Assert.assertTrue("root operator in operators", dagClone.getAllOperators().contains(dagClone.getRootOperators().get(0)));
+
+
+ Operator countGoodNodeClone = dagClone.getOperatorMeta("countGoodNode").getOperator();
+ Assert.assertEquals("", new Integer(10), dagClone.getContextAttributes(countGoodNodeClone).get(OperatorContext.SPIN_MILLIS));
+
+ }
+
+ @Test
+ public void testDeleteOperator()
+ {
+ LogicalPlan dag = new LogicalPlan();
+ TestGeneratorInputOperator input = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
+ GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
+ dag.addStream("s0", input.outport, o1.inport1);
+ StreamMeta s1 = dag.addStream("s1", o1.outport1, o2.inport1);
+ dag.validate();
+ Assert.assertEquals("", 3, dag.getAllOperators().size());
+
+ dag.removeOperator(o2);
+ s1.remove();
+ dag.validate();
+ Assert.assertEquals("", 2, dag.getAllOperators().size());
+ }
+
+ public static class ValidationTestOperator extends BaseOperator implements InputOperator {
+ @NotNull
+ @Pattern(regexp=".*malhar.*", message="Value has to contain 'malhar'!")
+ private String stringField1;
+
+ @Min(2)
+ private int intField1;
+
+ @AssertTrue(message="stringField1 should end with intField1")
+ private boolean isValidConfiguration() {
+ return stringField1.endsWith(String.valueOf(intField1));
+ }
+
+ private String getterProperty2 = "";
+
+ @NotNull
+ public String getProperty2() {
+ return getterProperty2;
+ }
+
+ public void setProperty2(String s) {
+ // annotations need to be on the getter
+ getterProperty2 = s;
+ }
+
+ private String[] stringArrayField;
+
+ public String[] getStringArrayField() {
+ return stringArrayField;
+ }
+
+ public void setStringArrayField(String[] stringArrayField) {
+ this.stringArrayField = stringArrayField;
+ }
+
+ public class Nested {
+ @NotNull
+ private String property = "";
+
+ public String getProperty() {
+ return property;
+ }
+
+ public void setProperty(String property) {
+ this.property = property;
+ }
+
+ }
+
+ @Valid
+ private final Nested nestedBean = new Nested();
+
+ private String stringProperty2;
+
+ public String getStringProperty2() {
+ return stringProperty2;
+ }
+
+ public void setStringProperty2(String stringProperty2) {
+ this.stringProperty2 = stringProperty2;
+ }
+
+ private Map<String, String> mapProperty = Maps.newHashMap();
+
+ public Map<String, String> getMapProperty()
+ {
+ return mapProperty;
+ }
+
+ public void setMapProperty(Map<String, String> mapProperty)
+ {
+ this.mapProperty = mapProperty;
+ }
+
+ @Override
+ public void emitTuples() {
+ // Emit no tuples
+
+ }
+
+ }
+
+ @Test
+ public void testOperatorValidation() {
+
+ ValidationTestOperator bean = new ValidationTestOperator();
+ bean.stringField1 = "malhar1";
+ bean.intField1 = 1;
+
+ // ensure validation standalone produces expected results
+ ValidatorFactory factory =
+ Validation.buildDefaultValidatorFactory();
+ Validator validator = factory.getValidator();
+ Set<ConstraintViolation<ValidationTestOperator>> constraintViolations =
+ validator.validate(bean);
+ //for (ConstraintViolation<ValidationTestOperator> cv : constraintViolations) {
+ // System.out.println("validation error: " + cv);
+ //}
+ Assert.assertEquals("" + constraintViolations,1, constraintViolations.size());
+ ConstraintViolation<ValidationTestOperator> cv = constraintViolations.iterator().next();
+ Assert.assertEquals("", bean.intField1, cv.getInvalidValue());
+ Assert.assertEquals("", "intField1", cv.getPropertyPath().toString());
+
+ // ensure DAG validation produces matching results
+ LogicalPlan dag = new LogicalPlan();
+ bean = dag.addOperator("testOperator", bean);
+
+ try {
+ dag.validate();
+ Assert.fail("should throw ConstraintViolationException");
+ } catch (ConstraintViolationException e) {
+ Assert.assertEquals("violation details", constraintViolations, e.getConstraintViolations());
+ String expRegex = ".*ValidationTestOperator\\{name=null}, propertyPath='intField1', message='must be greater than or equal to 2',.*value=1}]";
+ Assert.assertThat("exception message", e.getMessage(), RegexMatcher.matches(expRegex));
+ }
+
+ try {
+ bean.intField1 = 3;
+ dag.validate();
+ Assert.fail("should throw ConstraintViolationException");
+ } catch (ConstraintViolationException e) {
+ ConstraintViolation<?> cv2 = e.getConstraintViolations().iterator().next();
+ Assert.assertEquals("" + e.getConstraintViolations(), 1, constraintViolations.size());
+ Assert.assertEquals("", false, cv2.getInvalidValue());
+ Assert.assertEquals("", "validConfiguration", cv2.getPropertyPath().toString());
+ }
+ bean.stringField1 = "malhar3";
+
+ // annotated getter
+ try {
+ bean.getterProperty2 = null;
+ dag.validate();
+ Assert.fail("should throw ConstraintViolationException");
+ } catch (ConstraintViolationException e) {
+ ConstraintViolation<?> cv2 = e.getConstraintViolations().iterator().next();
+ Assert.assertEquals("" + e.getConstraintViolations(), 1, constraintViolations.size());
+ Assert.assertEquals("", null, cv2.getInvalidValue());
+ Assert.assertEquals("", "property2", cv2.getPropertyPath().toString());
+ }
+ bean.getterProperty2 = "";
+
+ // nested property
+ try {
+ bean.nestedBean.property = null;
+ dag.validate();
+ Assert.fail("should throw ConstraintViolationException");
+ } catch (ConstraintViolationException e) {
+ ConstraintViolation<?> cv2 = e.getConstraintViolations().iterator().next();
+ Assert.assertEquals("" + e.getConstraintViolations(), 1, constraintViolations.size());
+ Assert.assertEquals("", null, cv2.getInvalidValue());
+ Assert.assertEquals("", "nestedBean.property", cv2.getPropertyPath().toString());
+ }
+ bean.nestedBean.property = "";
+
+ // all valid
+ dag.validate();
+
+ }
+
+ @OperatorAnnotation(partitionable = false)
+ public static class TestOperatorAnnotationOperator extends BaseOperator {
+
+ @InputPortFieldAnnotation( optional = true)
+ final public transient DefaultInputPort<Object> input1 = new DefaultInputPort<Object>() {
+ @Override
+ public void process(Object tuple) {
+ }
+ };
+ }
+
+ class NoInputPortOperator extends BaseOperator {
+ }
+
+ @Test
+ public void testValidationForNonInputRootOperator() {
+ LogicalPlan dag = new LogicalPlan();
+ NoInputPortOperator x = dag.addOperator("x", new NoInputPortOperator());
+ try {
+ dag.validate();
+ Assert.fail("should fail because root operator is not input operator");
+ } catch (ValidationException e) {
+ // expected
+ }
+ }
+
+ @OperatorAnnotation(partitionable = false)
+ public static class TestOperatorAnnotationOperator2 extends BaseOperator implements Partitioner<TestOperatorAnnotationOperator2> {
+
+ @Override
+ public Collection<Partition<TestOperatorAnnotationOperator2>> definePartitions(Collection<Partition<TestOperatorAnnotationOperator2>> partitions, PartitioningContext context)
+ {
+ return null;
+ }
+
+ @Override
+ public void partitioned(Map<Integer, Partition<TestOperatorAnnotationOperator2>> partitions)
+ {
+ }
+ }
+
+ @Test
+ public void testOperatorAnnotation() {
+ LogicalPlan dag = new LogicalPlan();
+ TestGeneratorInputOperator input = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ TestOperatorAnnotationOperator operator = dag.addOperator("operator1", TestOperatorAnnotationOperator.class);
+ dag.addStream("Connection", input.outport, operator.input1);
+
+
+ dag.setAttribute(operator, OperatorContext.PARTITIONER, new StatelessPartitioner<TestOperatorAnnotationOperator>(2));
+
+ try {
+ dag.validate();
+ Assert.fail("should raise operator is not partitionable for operator1");
+ } catch (ValidationException e) {
+ Assert.assertEquals("", "Operator " + dag.getMeta(operator).getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!", e.getMessage());
+ }
+
+ dag.setAttribute(operator, OperatorContext.PARTITIONER, null);
+ dag.setInputPortAttribute(operator.input1, PortContext.PARTITION_PARALLEL, true);
+
+ try {
+ dag.validate();
+ Assert.fail("should raise operator is not partitionable for operator1");
+ } catch (ValidationException e) {
+ Assert.assertEquals("", "Operator " + dag.getMeta(operator).getName() + " is not partitionable but PARTITION_PARALLEL attribute is set", e.getMessage());
+ }
+
+ dag.setInputPortAttribute(operator.input1, PortContext.PARTITION_PARALLEL, false);
+ dag.validate();
+
+ dag.removeOperator(operator);
+ TestOperatorAnnotationOperator2 operator2 = dag.addOperator("operator2", TestOperatorAnnotationOperator2.class);
+
+ try {
+ dag.validate();
+ Assert.fail("should raise operator is not partitionable for operator2");
+ } catch (ValidationException e) {
+ Assert.assertEquals("Operator " + dag.getMeta(operator2).getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPortConnectionValidation() {
+
+ LogicalPlan dag = new LogicalPlan();
+
+ TestNonOptionalOutportInputOperator input = dag.addOperator("input1", TestNonOptionalOutportInputOperator.class);
+
+ try {
+ dag.validate();
+ Assert.fail("should raise port not connected for input1.outputPort1");
+
+ } catch (ValidationException e) {
+ Assert.assertEquals("", "Output port connection required: input1.outport1", e.getMessage());
+ }
+
+ GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
+ dag.addStream("stream1", input.outport1, o1.inport1);
+ dag.validate();
+
+ // required input
+ dag.addOperator("counter", CounterOperator.class);
+ try {
+ dag.validate();
+ } catch (ValidationException e) {
+ Assert.assertEquals("", "Input port connection required: counter.countInputPort", e.getMessage());
+ }
+
+ }
+
+ @Test
+ public void testAtMostOnceProcessingModeValidation() {
+ LogicalPlan dag = new LogicalPlan();
+
+ TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
+
+ GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class);
+ dag.setAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
+
+ dag.addStream("input1.outport", input1.outport, amoOper.inport1);
+ dag.addStream("input2.outport", input2.outport, amoOper.inport2);
+
+ GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class);
+ dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE);
+ dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1);
+
+ try {
+ dag.validate();
+ Assert.fail("Exception expected for " + outputOper);
+ } catch (ValidationException ve) {
+ Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/AT_MOST_ONCE");
+ }
+ dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, null);
+ dag.validate();
+
+ OperatorMeta outputOperOm = dag.getMeta(outputOper);
+ Assert.assertEquals("" + outputOperOm.getAttributes(), Operator.ProcessingMode.AT_MOST_ONCE, outputOperOm.getValue(OperatorContext.PROCESSING_MODE));
+
+ }
+
+ @Test
+ public void testExactlyOnceProcessingModeValidation() {
+ LogicalPlan dag = new LogicalPlan();
+
+ TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
+
+ GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class);
+ dag.setAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.EXACTLY_ONCE);
+
+ dag.addStream("input1.outport", input1.outport, amoOper.inport1);
+ dag.addStream("input2.outport", input2.outport, amoOper.inport2);
+
+ GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class);
+ dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1);
+
+ try {
+ dag.validate();
+ Assert.fail("Exception expected for " + outputOper);
+ } catch (ValidationException ve) {
+ Assert.assertEquals("", ve.getMessage(), "Processing mode for outputOper should be AT_MOST_ONCE for source amoOper/EXACTLY_ONCE");
+ }
+
+ dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE);
+
+ try {
+ dag.validate();
+ Assert.fail("Exception expected for " + outputOper);
+ } catch (ValidationException ve) {
+ Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/EXACTLY_ONCE");
+ }
+
+ // AT_MOST_ONCE is valid
+ dag.setAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
+ dag.validate();
+ }
+
+ @Test
+ public void testLocalityValidation() {
+ LogicalPlan dag = new LogicalPlan();
+
+ TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
+ StreamMeta s1 = dag.addStream("input1.outport", input1.outport, o1.inport1).setLocality(Locality.THREAD_LOCAL);
+ dag.validate();
+
+ TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
+ dag.addStream("input2.outport", input2.outport, o1.inport2);
+
+ try {
+ dag.validate();
+ Assert.fail("Exception expected for " + o1);
+ } catch (ValidationException ve) {
+ Assert.assertThat("", ve.getMessage(), RegexMatcher.matches("Locality THREAD_LOCAL invalid for operator .* with multiple input streams .*"));
+ }
+
+ s1.setLocality(null);
+ dag.validate();
+ }
+
+ private class TestAnnotationsOperator extends BaseOperator implements InputOperator {
+ //final public transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
+
+ @OutputPortFieldAnnotation( optional=false)
+ final public transient DefaultOutputPort<Object> outport2 = new DefaultOutputPort<Object>();
+
+ @Override
+ public void emitTuples() {
+ // Emit Nothing
+
+ }
+ }
+
+ private class TestAnnotationsOperator2 extends BaseOperator implements InputOperator{
+ // multiple ports w/o annotation, one of them must be connected
+ final public transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
+
+ @Override
+ public void emitTuples() {
+ // Emit Nothing
+
+ }
+ }
+
+ private class TestAnnotationsOperator3 extends BaseOperator implements InputOperator{
+ // multiple ports w/o annotation, one of them must be connected
+ @OutputPortFieldAnnotation( optional=true)
+ final public transient DefaultOutputPort<Object> outport1 = new DefaultOutputPort<Object>();
+ @OutputPortFieldAnnotation( optional=true)
+ final public transient DefaultOutputPort<Object> outport2 = new DefaultOutputPort<Object>();
+ @Override
+ public void emitTuples() {
+ // Emit Nothing
+
+ }
+ }
+
+ @Test
+ public void testOutputPortAnnotation() {
+ LogicalPlan dag = new LogicalPlan();
+ TestAnnotationsOperator ta1 = dag.addOperator("testAnnotationsOperator", new TestAnnotationsOperator());
+
+ try {
+ dag.validate();
+ Assert.fail("should raise: port connection required");
+ } catch (ValidationException e) {
+ Assert.assertEquals("", "Output port connection required: testAnnotationsOperator.outport2", e.getMessage());
+ }
+
+ TestOutputOperator o2 = dag.addOperator("sink", new TestOutputOperator());
+ dag.addStream("s1", ta1.outport2, o2.inport);
+
+ dag.validate();
+
+ TestAnnotationsOperator2 ta2 = dag.addOperator("multiOutputPorts1", new TestAnnotationsOperator2());
+
+ try {
+ dag.validate();
+ Assert.fail("should raise: At least one output port must be connected");
+ } catch (ValidationException e) {
+ Assert.assertEquals("", "At least one output port must be connected: multiOutputPorts1", e.getMessage());
+ }
+ TestOutputOperator o3 = dag.addOperator("o3", new TestOutputOperator());
+ dag.addStream("s2", ta2.outport1, o3.inport);
+
+ dag.addOperator("multiOutputPorts3", new TestAnnotationsOperator3());
+ dag.validate();
+
+ }
+
+ /**
+ * Operator that can be used with default Java serialization instead of Kryo
+ */
+ @DefaultSerializer(JavaSerializer.class)
+ public static class JdkSerializableOperator extends BaseOperator implements Serializable {
+ private static final long serialVersionUID = -4024202339520027097L;
+
+ public abstract class SerializableInputPort<T> implements InputPort<T>, Sink<T>, java.io.Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Sink<T> getSink() {
+ return this;
+ }
+
+ @Override
+ public void setConnected(boolean connected) {
+ }
+
+ @Override
+ public void setup(PortContext context)
+ {
+ }
+
+ @Override
+ public void teardown()
+ {
+ }
+
+ @Override
+ public StreamCodec<T> getStreamCodec() {
+ return null;
+ }
+ }
+
+ @InputPortFieldAnnotation( optional=true)
+ final public InputPort<Object> inport1 = new SerializableInputPort<Object>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ final public void put(Object payload)
+ {
+ }
+
+ @Override
+ public int getCount(boolean reset)
+ {
+ return 0;
+ }
+
+ };
+ }
+
+ @Test
+ public void testJdkSerializableOperator() throws Exception {
+ LogicalPlan dag = new LogicalPlan();
+ dag.addOperator("o1", new JdkSerializableOperator());
+
+ ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ LogicalPlan.write(dag, outStream);
+ outStream.close();
+
+ LogicalPlan clonedDag = LogicalPlan.read(new ByteArrayInputStream(outStream.toByteArray()));
+ JdkSerializableOperator o1Clone = (JdkSerializableOperator)clonedDag.getOperatorMeta("o1").getOperator();
+ Assert.assertNotNull("port object null", o1Clone.inport1);
+ }
+
+ private static class TestStreamCodec implements StreamCodec<Object> {
+ @Override
+ public Object fromByteArray(Slice fragment)
+ {
+ return fragment.stringValue();
+ }
+
+ @Override
+ public Slice toByteArray(Object o)
+ {
+ byte[] b = o.toString().getBytes();
+ return new Slice(b, 0, b.length);
+ }
+
+ @Override
+ public int getPartition(Object o)
+ {
+ return o.hashCode() / 2;
+ }
+ }
+
+ public static class TestPortCodecOperator extends BaseOperator {
+ public transient final DefaultInputPort<Object> inport1 = new DefaultInputPort<Object>()
+ {
+ @Override
+ public void process(Object tuple)
+ {
+
+ }
+
+ @Override
+ public StreamCodec<Object> getStreamCodec()
+ {
+ return new TestStreamCodec();
+ }
+ };
+
+ @OutputPortFieldAnnotation( optional = true)
+ public transient final DefaultOutputPort<Object> outport = new DefaultOutputPort<Object>();
+ }
+
+ /*
+ @Test
+ public void testStreamCodec() throws Exception {
+ LogicalPlan dag = new LogicalPlan();
+ TestGeneratorInputOperator input = dag.addOperator("input", TestGeneratorInputOperator.class);
+ GenericTestOperator gto1 = dag.addOperator("gto1", GenericTestOperator.class);
+ StreamMeta stream1 = dag.addStream("s1", input.outport, gto1.inport1);
+ StreamCodec<?> codec1 = new TestStreamCodec();
+ dag.setInputPortAttribute(gto1.inport1, PortContext.STREAM_CODEC, codec1);
+ dag.validate();
+ //Assert.assertEquals("Stream codec not set", stream1.getStreamCodec(), codec1);
+
+ GenericTestOperator gto2 = dag.addOperator("gto2", GenericTestOperator.class);
+ GenericTestOperator gto3 = dag.addOperator("gto3", GenericTestOperator.class);
+ StreamMeta stream2 = dag.addStream("s2", gto1.outport1, gto2.inport1, gto3.inport1);
+ dag.setInputPortAttribute(gto2.inport1, PortContext.STREAM_CODEC, codec1);
+ try {
+ dag.validate();
+ } catch (ValidationException e) {
+ String msg = e.getMessage();
+ if (!msg.startsWith("Stream codec not set on input port") || !msg.contains("gto3")
+ || !msg.contains(codec1.toString()) || !msg.endsWith("was specified on another port")) {
+ Assert.fail(String.format("LogicalPlan validation error msg: %s", msg));
+ }
+ }
+
+ dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec1);
+ dag.validate();
+ //Assert.assertEquals("Stream codec not set", stream2.getStreamCodec(), codec1);
+
+ StreamCodec<?> codec2 = new TestStreamCodec();
+ dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec2);
+ try {
+ dag.validate();
+ } catch (ValidationException e) {
+ String msg = e.getMessage();
+ if (!msg.startsWith("Conflicting stream codec set on input port") || !msg.contains("gto3")
+ || !msg.contains(codec2.toString()) || !msg.endsWith("was specified on another port")) {
+ Assert.fail(String.format("LogicalPlan validation error msg: %s", msg));
+ }
+ }
+
+ dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec1);
+ TestPortCodecOperator pco = dag.addOperator("pco", TestPortCodecOperator.class);
+ StreamMeta stream3 = dag.addStream("s3", gto2.outport1, pco.inport1);
+ dag.validate();
+ //Assert.assertEquals("Stream codec class not set", stream3.getCodecClass(), TestStreamCodec.class);
+
+ dag.setInputPortAttribute(pco.inport1, PortContext.STREAM_CODEC, codec2);
+ dag.validate();
+ //Assert.assertEquals("Stream codec not set", stream3.getStreamCodec(), codec2);
+ }
+ */
+
+ @Test
+ public void testCheckpointableWithinAppWindowAnnotation()
+ {
+ LogicalPlan dag = new LogicalPlan();
+ TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
+ dag.addStream("Stream1", input1.outport, x.inport1);
+ dag.setAttribute(x, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
+ dag.setAttribute(x, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
+ dag.validate();
+
+ TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
+ CheckpointableWithinAppWindowOperator y = dag.addOperator("y", new CheckpointableWithinAppWindowOperator());
+ dag.addStream("Stream2", input2.outport, y.inport1);
+ dag.setAttribute(y, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
+ dag.setAttribute(y, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
+ dag.validate();
+
+ TestGeneratorInputOperator input3 = dag.addOperator("input3", TestGeneratorInputOperator.class);
+ NotCheckpointableWithinAppWindowOperator z = dag.addOperator("z", new NotCheckpointableWithinAppWindowOperator());
+ dag.addStream("Stream3", input3.outport, z.inport1);
+ dag.setAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
+ dag.setAttribute(z, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
+ try {
+ dag.validate();
+ Assert.fail("should fail because chekpoint window count is not a factor of application window count");
+ }
+ catch (ValidationException e) {
+ // expected
+ }
+
+ dag.setAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 30);
+ dag.validate();
+
+ dag.setAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 45);
+ try {
+ dag.validate();
+ Assert.fail("should fail because chekpoint window count is not a factor of application window count");
+ }
+ catch (ValidationException e) {
+ // expected
+ }
+ }
+
+ @OperatorAnnotation(checkpointableWithinAppWindow = true)
+ class CheckpointableWithinAppWindowOperator extends GenericTestOperator
+ {
+ }
+
+ @OperatorAnnotation(checkpointableWithinAppWindow = false)
+ class NotCheckpointableWithinAppWindowOperator extends GenericTestOperator
+ {
+ }
+
+ @Test
+ public void testInputPortHiding()
+ {
+ LogicalPlan dag = new LogicalPlan();
+ TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ Operator2 operator2 = dag.addOperator("operator2", new Operator2());
+ dag.addStream("Stream1", input1.outport, operator2.input);
+ dag.validate();
+ }
+
+ @Test
+ public void testInvalidInputPortConnection()
+ {
+ LogicalPlan dag = new LogicalPlan();
+ TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
+ Operator1 operator1 = dag.addOperator("operator3", new Operator3());
+ dag.addStream("Stream1", input1.outport, operator1.input);
+ try {
+ dag.validate();
+ } catch (ValidationException ex) {
+ Assert.assertTrue("validation message", ex.getMessage().startsWith("Invalid port connected"));
+ return;
+ }
+ Assert.fail();
+ }
+
+ class Operator1 extends BaseOperator
+ {
+ public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
+ {
+ @Override
+ public void process(Object tuple)
+ {
+
+ }
+ };
+ }
+
+ class Operator2 extends Operator1
+ {
+ public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
+ {
+ @Override
+ public void process(Object tuple)
+ {
+
+ }
+ };
+ }
+
+ class Operator3 extends Operator1
+ {
+ @InputPortFieldAnnotation(optional = true)
+ public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
+ {
+ @Override
+ public void process(Object tuple)
+ {
+
+ }
+ };
+ }
+
+ @Test
+ public void testOutputPortHiding()
+ {
+ LogicalPlan dag = new LogicalPlan();
+ Operator5 operator5 = dag.addOperator("input", new Operator5());
+ Operator2 operator2 = dag.addOperator("operator2", new Operator2());
+ dag.addStream("Stream1", operator5.output, operator2.input);
+ dag.validate();
+ }
+
+ @Test(expected = ValidationException.class)
+ public void testInvalidOutputPortConnection()
+ {
+ LogicalPlan dag = new LogicalPlan();
+ Operator4 operator4 = dag.addOperator("input", new Operator5());
+ Operator3 operator3 = dag.addOperator("operator3", new Operator3());
+ dag.addStream("Stream1", operator4.output, operator3.input);
+ dag.validate();
+ }
+
+ class Operator4 extends BaseOperator implements InputOperator
+ {
+ public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
+
+ @Override
+ public void emitTuples()
+ {
+
+ }
+ }
+
+ class Operator5 extends Operator4
+ {
+ public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
+ }
+
+ /*
+ These were tests for operator semantics that verified if an operator class implements InputOperator then the same class should not declare input ports.
+ This would be done later when we are able to verify user code at compile-time.
+
+ validation()
+ {
+ if (n.getOperator() instanceof InputOperator) {
+ try {
+ for (Class<?> clazz : n.getOperator().getClass().getInterfaces()) {
+ if (clazz.getName().equals(InputOperator.class.getName())) {
+ for (Field field : n.getOperator().getClass().getDeclaredFields()) {
+ field.setAccessible(true);
+ Object declaredObject = field.get(n.getOperator());
+ if (declaredObject instanceof InputPort) {
+ throw new ValidationException("Operator class implements InputOperator and also declares input ports: " + n.name);
+ }
+ }
+ break;
+ }
+ }
+ }
+ catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ @Test
+ public void testInvalidInputOperatorDeclaration()
+ {
+ LogicalPlan dag = new LogicalPlan();
+
+ TestGeneratorInputOperator.InvalidInputOperator inputOperator = dag.addOperator("input", new TestGeneratorInputOperator.InvalidInputOperator());
+ GenericTestOperator operator2 = dag.addOperator("operator2", GenericTestOperator.class);
+
+ dag.addStream("stream1", inputOperator.outport, operator2.inport1);
+
+ try {
+ dag.validate();
+ fail("validation should fail");
+ }
+ catch (ValidationException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testValidInputOperatorDeclaration()
+ {
+ LogicalPlan dag = new LogicalPlan();
+
+ TestGeneratorInputOperator.ValidGenericOperator operator1 = dag.addOperator("input", new TestGeneratorInputOperator.ValidGenericOperator());
+ GenericTestOperator operator2 = dag.addOperator("operator2", GenericTestOperator.class);
+
+ dag.addStream("stream1", operator1.outport, operator2.inport1);
+ dag.validate();
+ }
+ */
+}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/977093e1/engine/src/test/resources/schemaTestTopology.json
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/schemaTestTopology.json b/engine/src/test/resources/schemaTestTopology.json
index 6c779fd..46c0e78 100644
--- a/engine/src/test/resources/schemaTestTopology.json
+++ b/engine/src/test/resources/schemaTestTopology.json
@@ -36,7 +36,7 @@
}
],
"schema": {
- "class": "com.datatorrent.stram.plan.LogicalPlanConfigurationTest$TestSchema"
+ "class": "com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest$TestSchema"
}
}
]
[2/8] incubator-apex-core git commit: APEX-28 #resolve
Posted by cs...@apache.org.
APEX-28 #resolve
- Added the ability to specify attributes in a properties xml as dt.attr.MY_ATTR
- Added handling for ambiguous attributes
- Added the ability to specify attributes for operators and ports on parent elements like applications
- Cleaned up commented out code and warnings in LogicalPlanConfiguration
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/434a7170
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/434a7170
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/434a7170
Branch: refs/heads/devel-3
Commit: 434a7170e4a4e2da013e283706b5bf42f853f6a4
Parents: d748ed4
Author: Timothy Farkas <ti...@datatorrent.com>
Authored: Tue Aug 11 10:50:57 2015 -0700
Committer: Timothy Farkas <ti...@datatorrent.com>
Committed: Sun Aug 30 18:19:40 2015 -0700
----------------------------------------------------------------------
.../java/com/datatorrent/api/Attribute.java | 11 +-
.../main/java/com/datatorrent/api/Context.java | 10 +
.../stram/plan/logical/LogicalPlan.java | 8 +-
.../plan/logical/LogicalPlanConfiguration.java | 1145 ++++++++++++++----
.../plan/LogicalPlanConfigurationTest.java | 1034 +++++++++++++++-
.../datatorrent/stram/plan/LogicalPlanTest.java | 2 +
.../stram/plan/logical/MockStorageAgent.java | 67 +
engine/src/test/resources/testTopology.json | 4 +-
8 files changed, 2008 insertions(+), 273 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/api/src/main/java/com/datatorrent/api/Attribute.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Attribute.java b/api/src/main/java/com/datatorrent/api/Attribute.java
index a7492b5..4c16a2a 100644
--- a/api/src/main/java/com/datatorrent/api/Attribute.java
+++ b/api/src/main/java/com/datatorrent/api/Attribute.java
@@ -277,6 +277,13 @@ public class Attribute<T> implements Serializable
if (map.containsKey(clazz)) {
return 0;
}
+
+ map.put(clazz, getAttributesNoSave(clazz));
+ return (long)clazz.getModifiers() << 32 | clazz.hashCode();
+ }
+
+ public static Set<Attribute<Object>> getAttributesNoSave(Class<?> clazz)
+ {
Set<Attribute<Object>> set = new HashSet<Attribute<Object>>();
try {
for (Field f: clazz.getDeclaredFields()) {
@@ -323,8 +330,8 @@ public class Attribute<T> implements Serializable
catch (Exception ex) {
DTThrowable.rethrow(ex);
}
- map.put(clazz, set);
- return (long)clazz.getModifiers() << 32 | clazz.hashCode();
+
+ return set;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/api/src/main/java/com/datatorrent/api/Context.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/com/datatorrent/api/Context.java b/api/src/main/java/com/datatorrent/api/Context.java
index cd10398..c2d974a 100644
--- a/api/src/main/java/com/datatorrent/api/Context.java
+++ b/api/src/main/java/com/datatorrent/api/Context.java
@@ -33,6 +33,16 @@ import com.datatorrent.api.annotation.Stateless;
*/
public interface Context
{
+ /*
+ * Note: If the same name is given to an Attribute specified in multiple Context classes, then the type of that
+ * Attribute is required to be the same accross all Context classes. This is required because if a simple attribute
+ * name is specified in a properties file at the top level context then that attribute needs to be set in all child configurations. If
+ * there were multiple Attributes specified in different Contexts with the same name, but a different type, then
+ * it would not be possible to set the values of Attributes specified by a simple attribute name in the root
+ * context of a properties file. If this were the case, then adding another Attribute with the same name as a pre-existing Attribute to a new Context
+ * class would be a backwards incompatible change.
+ */
+
/**
* Get the attributes associated with this context.
* The returned map does not contain any attributes that may have been defined in the parent context of this context.
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
index 6741d37..8826896 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
@@ -1430,8 +1430,12 @@ public class LogicalPlan implements Serializable, DAG
}
for (StreamMeta s: streams.values()) {
- if (s.source == null || (s.sinks.isEmpty())) {
- throw new ValidationException(String.format("stream not connected: %s", s.getName()));
+ if (s.source == null) {
+ throw new ValidationException(String.format("stream source not connected: %s", s.getName()));
+ }
+
+ if (s.sinks.isEmpty()) {
+ throw new ValidationException(String.format("stream sink not connected: %s", s.getName()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/434a7170/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
index 46291a8..a3a18c2 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlanConfiguration.java
@@ -20,12 +20,17 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.Map.Entry;
+import jline.internal.Preconditions;
+
import javax.validation.ValidationException;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -37,6 +42,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.beanutils.BeanUtils;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;
import org.apache.commons.lang3.StringUtils;
@@ -44,6 +50,7 @@ import org.apache.hadoop.conf.Configuration;
import com.datatorrent.api.*;
import com.datatorrent.api.Attribute.AttributeMap.AttributeInitializer;
+import com.datatorrent.api.Context.DAGContext;
import com.datatorrent.api.Context.OperatorContext;
import com.datatorrent.api.Context.PortContext;
import com.datatorrent.api.annotation.ApplicationAnnotation;
@@ -54,6 +61,7 @@ import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta;
import com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta;
+import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.StramElement;
import com.datatorrent.stram.util.ObjectMapperFactory;
/**
@@ -88,30 +96,51 @@ public class LogicalPlanConfiguration {
public static final String TEMPLATE_classNameRegExp = "matchClassNameRegExp";
public static final String CLASS = "class";
+ public static final String KEY_SEPARATOR = ".";
+ public static final String KEY_SEPARATOR_SPLIT_REGEX = "\\.";
private static final String CLASS_SUFFIX = "." + CLASS;
private static final String WILDCARD = "*";
private static final String WILDCARD_PATTERN = ".*";
+ /**
+ * This is done to initialize the serial id of these interfaces.
+ */
static {
Object serial[] = new Object[] {Context.DAGContext.serialVersionUID, OperatorContext.serialVersionUID, PortContext.serialVersionUID};
LOG.debug("Initialized attributes {}", serial);
}
- private enum StramElement {
+ /**
+ * This represents an element that can be referenced in a DT property.
+ */
+ protected enum StramElement {
APPLICATION("application"), GATEWAY("gateway"), TEMPLATE("template"), OPERATOR("operator"),STREAM("stream"), PORT("port"), INPUT_PORT("inputport"),OUTPUT_PORT("outputport"),
ATTR("attr"), PROP("prop"),CLASS("class"),PATH("path");
private final String value;
+ /**
+ * Creates a {@link StramElement} with the corresponding name.
+ * @param value The name of the {@link StramElement}.
+ */
StramElement(String value) {
this.value = value;
}
+ /**
+ * Gets the name of the {@link StramElement}.
+ * @return The name of the {@link StramElement}.
+ */
public String getValue() {
return value;
}
+ /**
+ * Gets the {@link StramElement} corresponding to the given name.
+ * @param value The name for which a {@link StramElement} is desired.
+ * @return The {@link StramElement} corresponding to the given name.
+ */
public static StramElement fromValue(String value) {
StramElement velement = null;
for (StramElement element : StramElement.values()) {
@@ -124,7 +153,754 @@ public class LogicalPlanConfiguration {
}
}
-
+
+ /**
+ * This is an enum which represents a type of configuration.
+ */
+ protected enum ConfElement
+ {
+ @SuppressWarnings("SetReplaceableByEnumSet")
+ STRAM(null,
+ null,
+ new HashSet<StramElement>(),
+ null),
+ @SuppressWarnings("SetReplaceableByEnumSet")
+ APPLICATION(StramElement.APPLICATION,
+ STRAM,
+ new HashSet<StramElement>(),
+ DAGContext.class),
+ @SuppressWarnings("SetReplaceableByEnumSet")
+ TEMPLATE(StramElement.TEMPLATE,
+ STRAM,
+ new HashSet<StramElement>(),
+ null),
+ @SuppressWarnings("SetReplaceableByEnumSet")
+ GATEWAY(StramElement.GATEWAY,
+ ConfElement.APPLICATION,
+ new HashSet<StramElement>(),
+ null),
+ @SuppressWarnings("SetReplaceableByEnumSet")
+ OPERATOR(StramElement.OPERATOR,
+ ConfElement.APPLICATION,
+ new HashSet<StramElement>(),
+ OperatorContext.class),
+ @SuppressWarnings("SetReplaceableByEnumSet")
+ STREAM(StramElement.STREAM,
+ ConfElement.APPLICATION,
+ new HashSet<StramElement>(),
+ null),
+ PORT(StramElement.PORT,
+ ConfElement.OPERATOR,
+ Sets.newHashSet(StramElement.INPUT_PORT, StramElement.OUTPUT_PORT),
+ PortContext.class);
+
+ public static final Map<StramElement, ConfElement> STRAM_ELEMENT_TO_CONF_ELEMENT = Maps.newHashMap();
+ public static final Map<Class<? extends Context>, ConfElement> CONTEXT_TO_CONF_ELEMENT = Maps.newHashMap();
+
+ static {
+ initialize();
+ }
+
+ protected static void initialize()
+ {
+ STRAM.setChildren(Sets.newHashSet(APPLICATION, TEMPLATE));
+ APPLICATION.setChildren(Sets.newHashSet(GATEWAY, OPERATOR, STREAM));
+ OPERATOR.setChildren(Sets.newHashSet(PORT));
+
+ STRAM_ELEMENT_TO_CONF_ELEMENT.clear();
+
+ //Initialize StramElement to ConfElement
+ for (ConfElement confElement: ConfElement.values()) {
+ STRAM_ELEMENT_TO_CONF_ELEMENT.put(confElement.getStramElement(), confElement);
+
+ for (StramElement sElement: confElement.getAllRelatedElements()) {
+ STRAM_ELEMENT_TO_CONF_ELEMENT.put(sElement, confElement);
+ }
+ }
+
+ //Initialize attributes
+ for (ConfElement confElement: ConfElement.values()) {
+ if (confElement.getParent() == null) {
+ continue;
+ }
+
+ setAmbiguousAttributes(confElement);
+ }
+
+ // build context to conf element map
+ CONTEXT_TO_CONF_ELEMENT.clear();
+
+ for (ConfElement confElement: ConfElement.values()) {
+ CONTEXT_TO_CONF_ELEMENT.put(confElement.getContextClass(), confElement);
+ }
+
+ //Check if all the context classes are accounted for
+ Set<Class<? extends Context>> confElementContextClasses = Sets.newHashSet();
+
+ for (ConfElement confElement: ConfElement.values()) {
+ if (confElement.getContextClass() == null) {
+ continue;
+ }
+
+ confElementContextClasses.add(confElement.getContextClass());
+ }
+
+ if (!ContextUtils.CONTEXT_CLASSES.equals(confElementContextClasses)) {
+ throw new IllegalStateException("All the context classes "
+ + ContextUtils.CONTEXT_CLASSES
+ + " found in "
+ + Context.class
+ + " are not used by ConfElements "
+ + confElementContextClasses);
+ }
+ }
+
+ /**
+ * This is a recursive method to initialize the ambiguous elements for each
+ * {@link ConfElement}.
+ *
+ * @param element The current {@link ConfElement} at which to start initializing
+ * the ambiguous elements.
+ * @return The set of all simple attribute names encountered up to this point.
+ */
+ public static Set<String> setAmbiguousAttributes(ConfElement element)
+ {
+ Set<String> ambiguousAttributes = Sets.newHashSet();
+ Set<String> allChildAttributes = Sets.newHashSet(element.getContextAttributes());
+
+ for (ConfElement childElement: element.getChildren()) {
+ Set<String> allAttributes = setAmbiguousAttributes(childElement);
+ ambiguousAttributes.addAll(childElement.getAmbiguousAttributes());
+
+ @SuppressWarnings("unchecked")
+ Set<String> intersection = (Set<String>)Sets.newHashSet(CollectionUtils.intersection(allChildAttributes,
+ allAttributes));
+ ambiguousAttributes.addAll(intersection);
+ allChildAttributes.addAll(allAttributes);
+ }
+
+ element.setAmbiguousAttributes(ambiguousAttributes);
+ element.setAllChildAttributes(allChildAttributes);
+
+ return allChildAttributes;
+ }
+
+ private final StramElement element;
+ private final ConfElement parent;
+ private Set<ConfElement> children = Sets.newHashSet();
+ private final Set<StramElement> allRelatedElements = Sets.newHashSet();
+ private final Class<? extends Context> contextClass;
+ private Set<String> ambiguousAttributes = Sets.newHashSet();
+ private Set<String> contextAttributes = Sets.newHashSet();
+ private Set<String> allChildAttributes = Sets.newHashSet();
+
+ /**
+ * This creates a {@link ConfElement}.
+ *
+ * @param element The current {@link StramElement} representing a {@link ConfElement}.
+ * @param parent The parent {@link ConfElement}.
+ * @param additionalRelatedElements Any additional {@link StramElement} that could be
+ * related to this {@link ConfElement}.
+ * @param contextClass The {@link Context} class that contains all the attributes to
+ * be used by this {@link ConfElement}.
+ */
+ ConfElement(StramElement element,
+ ConfElement parent,
+ Set<StramElement> additionalRelatedElements,
+ Class<? extends Context> contextClass)
+ {
+ this.element = element;
+ this.parent = parent;
+
+ this.allRelatedElements.addAll(additionalRelatedElements);
+ this.allRelatedElements.add(element);
+
+ this.contextClass = contextClass;
+
+ if (contextClass != null) {
+ this.contextAttributes = ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass);
+ } else {
+ this.contextAttributes = Sets.newHashSet();
+ }
+ }
+
+ private void setAllChildAttributes(Set<String> allChildAttributes)
+ {
+ this.allChildAttributes = Preconditions.checkNotNull(allChildAttributes);
+ }
+
+ public Set<String> getAllChildAttributes()
+ {
+ return allChildAttributes;
+ }
+
+ private void setAmbiguousAttributes(Set<String> ambiguousAttributes)
+ {
+ this.ambiguousAttributes = Preconditions.checkNotNull(ambiguousAttributes);
+ }
+
+ /**
+ * Gets the simple names of attributes which are specified under multiple configurations which
+ * include this configuration or any child configurations.
+ *
+ * @return The set of ambiguous simple attribute names.
+ */
+ public Set<String> getAmbiguousAttributes()
+ {
+ return ambiguousAttributes;
+ }
+
+ /**
+ * Gets the {@link Context} class that corresponds to this {@link ConfElement}.
+ *
+ * @return The {@link Context} class that corresponds to this {@link ConfElement}.
+ */
+ public Class<? extends Context> getContextClass()
+ {
+ return contextClass;
+ }
+
+ /**
+ * Gets the {@link StramElement} representing this {@link ConfElement}.
+ *
+ * @return The {@link StramElement} corresponding to this {@link ConfElement}.
+ */
+ public StramElement getStramElement()
+ {
+ return element;
+ }
+
+ /**
+ * Gets the attributes contained in the {@link Context} associated with this {@link ConfElement}.
+ *
+ * @return A {@link java.util.Set} containing the simple attribute names of all of the attributes
+ * contained in the {@link Context} associated with this {@link ConfElement}.
+ */
+ public Set<String> getContextAttributes()
+ {
+ return contextAttributes;
+ }
+
+ /**
+ * Gets the {@link ConfElement} that is the parent of this {@link ConfElement}.
+ *
+ * @return The {@link ConfElement} that is the parent of this {@link ConfElement}.
+ */
+ public ConfElement getParent()
+ {
+ return parent;
+ }
+
+ /**
+ * Sets the child {@link ConfElement}s of this {@link ConfElement}.
+ *
+ * @param children The child {@link ConfElement}s of this {@link ConfElement}.
+ */
+ private void setChildren(Set<ConfElement> children)
+ {
+ this.children = Preconditions.checkNotNull(children);
+ }
+
+ /**
+ * Gets the child {@link ConfElement}s of this {@link ConfElement}.
+ *
+ * @return The child {@link ConfElement} of this {@link ConfElement}
+ */
+ public Set<ConfElement> getChildren()
+ {
+ return children;
+ }
+
+ /**
+ * Gets all the {@link StramElement}s that are represented by this {@link ConfElement}.
+ *
+ * @return All the {@link StramElement}s that are represented by this {@link ConfElement}.
+ */
+ public Set<StramElement> getAllRelatedElements()
+ {
+ return allRelatedElements;
+ }
+
+ /**
+ * Gets the {@link StramElement} representing the {@link Conf} type which can be a parent of the {@link Conf} type
+ * represented by the given {@link StramElement}.
+ *
+ * @param conf The {@link StramElement} representing the {@link Conf} type of interest.
+ * @return The {@link StramElement} representing the {@link Conf} type which can be a parent of the given {@link Conf} type.
+ */
+ public static StramElement getAllowedParentConf(StramElement conf)
+ {
+ ConfElement confElement = STRAM_ELEMENT_TO_CONF_ELEMENT.get(conf);
+
+ if (confElement == null) {
+ throw new IllegalArgumentException(conf + " is not a valid conf element.");
+ }
+
+ return confElement.getParent().getStramElement();
+ }
+
+ /**
+ * Creates a list of {@link StramElement}s which represent the path from the current {@link Conf} type to
+ * a root {@link Conf} type. This path includes the current {@link Conf} type as well as the root.
+ *
+ * @param conf The current {@link Conf} type.
+ * @return A path from the current {@link Conf} type to a root {@link Conf} type, which includes the current and root
+ * {
+ * @lin Conf} types.
+ */
+ public static List<StramElement> getPathFromChildToRootInclusive(StramElement conf)
+ {
+ ConfElement confElement = STRAM_ELEMENT_TO_CONF_ELEMENT.get(conf);
+
+ if (confElement == null) {
+ throw new IllegalArgumentException(conf + " does not represent a valid configuration type.");
+ }
+
+ List<StramElement> path = Lists.newArrayList();
+
+ for (; confElement != null; confElement = confElement.getParent()) {
+ path.add(confElement.getStramElement());
+ }
+
+ return path;
+ }
+
+ /**
+ * Creates a list of {@link StramElement}s which represent the path from the root {@link Conf} type to
+ * the current {@link Conf} type. This path includes the root {@link Conf} type as well as the current {@link Conf} type.
+ *
+ * @param conf The current {@link Conf} type.
+ * @return A path from the root {@link Conf} type to the current {@link Conf} type, which includes the current and root
+ * {
+ * @lin Conf} types.
+ */
+ public static List<StramElement> getPathFromRootToChildInclusive(StramElement conf)
+ {
+ List<StramElement> path = getPathFromChildToRootInclusive(conf);
+ return Lists.reverse(path);
+ }
+
+ /**
+ * Creates a list of {@link StramElement}s which represent the path from the current {@link Conf} type to
+ * a parent {@link Conf} type. This path includes the current {@link Conf} type as well as the parent.
+ *
+ * @param child The current {@link Conf} type.
+ * @param parent The parent {@link Conf} type.
+ * @return A path from the current {@link Conf} type to a parent {@link Conf} type, which includes the current and parent
+ * {
+ * @lin Conf} types.
+ */
+ public static List<StramElement> getPathFromChildToParentInclusive(StramElement child,
+ StramElement parent)
+ {
+ ConfElement confElement = STRAM_ELEMENT_TO_CONF_ELEMENT.get(child);
+
+ if (confElement == null) {
+ throw new IllegalArgumentException(child + " does not represent a valid configuration type.");
+ }
+
+ List<StramElement> path = Lists.newArrayList();
+
+ if (child == parent) {
+ path.add(child);
+ return path;
+ }
+
+ for (; confElement != null; confElement = confElement.getParent()) {
+ path.add(confElement.getStramElement());
+
+ if (confElement.getStramElement() == parent) {
+ break;
+ }
+ }
+
+ if (path.get(path.size() - 1) != parent) {
+ throw new IllegalArgumentException(parent + " is not a valid parent of " + child);
+ }
+
+ return path;
+ }
+
+ /**
+ * Creates a list of {@link StramElement}s which represent the path from the parent {@link Conf} type to
+ * a child {@link Conf} type. This path includes the parent {@link Conf} type as well as the current {@link Conf} type.
+ *
+ * @param child The current {@link Conf} type.
+ * @param parent The parent {@link Conf} type.
+ * @return A path from the parent {@link Conf} type to the current {@link Conf} type, which includes the current and parent
+ * {
+ * @lin Conf} types.
+ */
+ public static List<StramElement> getPathFromParentToChildInclusive(StramElement child,
+ StramElement parent)
+ {
+ List<StramElement> path = getPathFromChildToParentInclusive(child,
+ parent);
+ return Lists.reverse(path);
+ }
+
+ /**
+ * This method searches the current {@link ConfElement} and its children to find a {@link ConfElement}
+ * that contains the given simple {@link Attribute} name.
+ *
+ * @param current The current {@link ConfElement}.
+ * @param simpleAttributeName The simple {@link Attribute} name to search for.
+ * @return The {@link ConfElement} that contains the given attribute, or null if no {@link ConfElement} contains
+ * the given attribute.
+ */
+ public static ConfElement findConfElementWithAttribute(ConfElement current,
+ String simpleAttributeName)
+ {
+ if (current.getContextAttributes().contains(simpleAttributeName)) {
+ return current;
+ }
+
+ for (ConfElement childConfElement: current.getChildren()) {
+ ConfElement result = findConfElementWithAttribute(childConfElement,
+ simpleAttributeName);
+
+ if (result != null) {
+ return result;
+ }
+ }
+
+ return null;
+ }
+
+ protected static Conf addConfs(Conf parentConf, ConfElement childConfElement)
+ {
+ //Figure out what configurations need to be added to hold this attribute
+ List<StramElement> path = ConfElement.getPathFromParentToChildInclusive(childConfElement.getStramElement(),
+ parentConf.getConfElement().getStramElement());
+
+ for (int pathIndex = 1;
+ pathIndex < path.size();
+ pathIndex++) {
+ LOG.debug("Adding conf");
+ StramElement pathElement = path.get(pathIndex);
+ //Add the configurations we need to hold this attribute
+ parentConf = addConf(pathElement, WILDCARD, parentConf);
+ }
+
+ return parentConf;
+ }
+
+ }
+
+ /**
+ * Utility class that holds methods for handling {@link Context} classes.
+ */
+ @SuppressWarnings("unchecked")
+ protected static class ContextUtils
+ {
+ public static final Map<Class<? extends Context>, Set<String>> CONTEXT_CLASS_TO_ATTRIBUTES;
+ public static final Set<Class<? extends Context>> CONTEXT_CLASSES;
+ public static final Map<Class<? extends Context>, Map<String, Attribute<?>>> CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE;
+
+ static {
+ CONTEXT_CLASSES = Sets.newHashSet();
+
+ for (Class<?> clazz: Context.class.getDeclaredClasses()) {
+ if (!Context.class.isAssignableFrom(clazz)) {
+ continue;
+ }
+
+ CONTEXT_CLASSES.add((Class<? extends Context>)clazz);
+ }
+
+ CONTEXT_CLASS_TO_ATTRIBUTES = Maps.newHashMap();
+
+ for (Class<? extends Context> contextClass: CONTEXT_CLASSES) {
+ Set<String> contextAttributes = Sets.newHashSet();
+
+ Field[] fields = contextClass.getDeclaredFields();
+
+ for (Field field: fields) {
+ if (!Attribute.class.isAssignableFrom(field.getType())) {
+ continue;
+ }
+
+ contextAttributes.add(field.getName());
+ }
+
+ CONTEXT_CLASS_TO_ATTRIBUTES.put(contextClass, contextAttributes);
+ }
+
+ CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE = Maps.newHashMap();
+
+ for (Class<? extends Context> contextClass: CONTEXT_CLASSES) {
+ Map<String, Attribute<?>> simpleAttributeNameToAttribute = Maps.newHashMap();
+ CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.put(contextClass, simpleAttributeNameToAttribute);
+
+ Set<Attribute<Object>> attributes = AttributeInitializer.getAttributesNoSave(contextClass);
+
+ LOG.debug("context class {} and attributes {}", contextClass, attributes);
+
+ for (Attribute<Object> attribute: attributes) {
+ simpleAttributeNameToAttribute.put(AttributeParseUtils.getSimpleName(attribute), attribute);
+ }
+ }
+ }
+
+ private ContextUtils()
+ {
+ }
+
+ /**
+ * This method is only used for testing.
+ *
+ * @param contextClass
+ * @param attribute
+ */
+ @VisibleForTesting
+ protected static void addAttribute(Class<? extends Context> contextClass, Attribute<?> attribute)
+ {
+ Set<String> attributeNames = CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass);
+
+ if (attributeNames == null) {
+ attributeNames = Sets.newHashSet();
+ CONTEXT_CLASS_TO_ATTRIBUTES.put(contextClass, attributeNames);
+ }
+
+ attributeNames.add(attribute.getSimpleName());
+
+ CONTEXT_CLASSES.add(contextClass);
+ Map<String, Attribute<?>> attributeMap = CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(contextClass);
+
+ if (attributeMap == null) {
+ attributeMap = Maps.newHashMap();
+ CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.put(contextClass, attributeMap);
+ }
+
+ attributeMap.put(attribute.getSimpleName(), attribute);
+ }
+
+ /**
+ * This method is only used for testing.
+ *
+ * @param contextClass
+ * @param attribute
+ */
+ @VisibleForTesting
+ protected static void removeAttribute(Class<? extends Context> contextClass, Attribute<?> attribute)
+ {
+ Set<String> attributeNames = CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass);
+
+ if (attributeNames != null) {
+ attributeNames.remove(attribute.getSimpleName());
+
+ if (attributeNames.isEmpty()) {
+ CONTEXT_CLASS_TO_ATTRIBUTES.remove(contextClass);
+ }
+ }
+
+ if (!CONTEXT_CLASS_TO_ATTRIBUTES.keySet().contains(contextClass)) {
+ CONTEXT_CLASSES.remove(contextClass);
+ }
+
+ Map<String, Attribute<?>> attributeMap = CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(contextClass);
+
+ if (attributeMap != null) {
+ attributeMap.remove(attribute.getSimpleName());
+
+ if (attributeMap.isEmpty()) {
+ CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.remove(contextClass);
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Utility class that holds methods for parsing.
+ */
+ protected static class AttributeParseUtils
+ {
+ public static final Set<String> ALL_SIMPLE_ATTRIBUTE_NAMES;
+
+ static {
+ ALL_SIMPLE_ATTRIBUTE_NAMES = Sets.newHashSet();
+
+ initialize();
+ }
+
+ public static void initialize()
+ {
+ ALL_SIMPLE_ATTRIBUTE_NAMES.clear();
+
+ for (Map.Entry<Class<? extends Context>, Set<String>> entry: ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.entrySet()) {
+ ALL_SIMPLE_ATTRIBUTE_NAMES.addAll(entry.getValue());
+ }
+ }
+
+ private AttributeParseUtils()
+ {
+ }
+
+ /**
+ * This method creates all the appropriate child {@link Conf}s of the given parent {@link Conf} and adds the given
+ * attribute to the parent {@link Conf} if appropriate as well as all the child {@link Conf}s of the parent if
+ * appropriate.
+ *
+ * @param conf The parent {@link Conf}.
+ * @param attributeName The simple name of the attribute to add.
+ * @param attrValue The value of the attribute.
+ */
+ protected static void processAllConfsForAttribute(Conf conf, String attributeName, String attrValue)
+ {
+ ConfElement confElement = conf.getConfElement();
+
+ LOG.debug("Current confElement {} and name {}", confElement.getStramElement(), conf.getId());
+
+ if (confElement.getContextAttributes().contains(attributeName)) {
+ LOG.debug("Adding attribute");
+ @SuppressWarnings("unchecked")
+ Attribute<Object> attr = (Attribute<Object>)ContextUtils.CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(confElement.getContextClass()).get(attributeName);
+ conf.setAttribute(attr, attrValue);
+ }
+
+ for (ConfElement childConfElement: confElement.getChildren()) {
+
+ if (!childConfElement.getAllChildAttributes().contains(attributeName)) {
+ continue;
+ }
+
+ Conf childConf = addConf(childConfElement.getStramElement(), WILDCARD, conf);
+ processAllConfsForAttribute(childConf, attributeName, attrValue);
+ }
+ }
+
+ /**
+ * This extracts the name of an attribute from the given set of keys.
+ *
+ * @param element The {@link StramElement} corresponding to the current element being parsed.
+ * @param keys The split keys that are being parse.
+ * @param index The current key that the parser is on.
+ * @return The FQN name of an attribute or just the name of an Attribute.
+ */
+ public static String getAttributeName(StramElement element, String[] keys, int index)
+ {
+
+ if (element != null && element != StramElement.ATTR) {
+ throw new IllegalArgumentException("The given "
+ + StramElement.class
+ + " must either have a value of null or "
+ + StramElement.ATTR
+ + " but it had a value of " + element);
+ }
+
+ String attributeName;
+
+ if (element == StramElement.ATTR) {
+ attributeName = getCompleteKey(keys, index + 1);
+ } else {
+ attributeName = getCompleteKey(keys, index);
+ }
+
+ return attributeName;
+ }
+
+ /**
+ * This method checks to see if the attribute name is simple or is prefixed with the FQCN of the {@link Context}
+ * class which contains it.
+ *
+ * @param attributeName The attribute name to check.
+ * @return True if the attribute name is simple. False otherwise.
+ */
+ public static boolean isSimpleAttributeName(String attributeName)
+ {
+ return !attributeName.contains(KEY_SEPARATOR);
+ }
+
+ /**
+ * Gets the {@link Context} class that the given attributeName belongs to.
+ *
+ * @param attributeName The {@link Attribute} name whose {@link Context} class needs to be
+ * discovered.
+ * @return The {@link Context} class that the given {@link Attribute} name belongs to.
+ */
+ @SuppressWarnings("unchecked")
+ public static Class<? extends Context> getContainingContextClass(String attributeName)
+ {
+ if (isSimpleAttributeName(attributeName)) {
+ throw new IllegalArgumentException("The given attribute name "
+ + attributeName
+ + " is simple.");
+ }
+
+ LOG.debug("Attribute Name {}", attributeName);
+
+ int lastSeparator = attributeName.lastIndexOf(KEY_SEPARATOR);
+ String contextClassName = attributeName.substring(0, lastSeparator);
+
+ int lastPeriod = contextClassName.lastIndexOf(KEY_SEPARATOR);
+
+ StringBuilder sb = new StringBuilder(contextClassName);
+ sb.setCharAt(lastPeriod, '$');
+ contextClassName = sb.toString();
+
+ Class<? extends Context> contextClass;
+
+ try {
+ Class<?> clazz = Class.forName(contextClassName);
+
+ if (Context.class.isAssignableFrom(clazz)) {
+ contextClass = (Class<? extends Context>)clazz;
+ } else {
+ throw new IllegalArgumentException("The provided context class name "
+ + contextClassName
+ + " is not valid.");
+ }
+ } catch (ClassNotFoundException ex) {
+ throw new IllegalArgumentException(ex);
+ }
+
+ String simpleAttributeName = getSimpleAttributeName(attributeName);
+
+ if (!ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass).contains(simpleAttributeName)) {
+ throw new ValidationException(simpleAttributeName
+ + " is not a valid attribute of "
+ + contextClass);
+ }
+
+ return contextClass;
+ }
+
+ /**
+ * This extract this simple {@link Attribute} name from the given {@link Attribute} name.
+ *
+ * @param attributeName The attribute name to extract a simple attribute name from.
+ * @return The simple attribute name.
+ */
+ public static String getSimpleAttributeName(String attributeName)
+ {
+ if (isSimpleAttributeName(attributeName)) {
+ return attributeName;
+ }
+
+ if (attributeName.endsWith(KEY_SEPARATOR)) {
+ throw new IllegalArgumentException("The given attribute name ends with \""
+ + KEY_SEPARATOR
+ + "\" so a simple name cannot be extracted.");
+ }
+
+ return attributeName.substring(attributeName.lastIndexOf(KEY_SEPARATOR) + 1, attributeName.length());
+ }
+
+ /**
+ * Gets the simple name of an {@link Attribute}, which does not include the FQCN of the {@link Context} class
+ * which contains it.
+ *
+ * @param attribute The {@link Attribute} of interest.
+ * @return The name of an {@link Attribute}.
+ */
+ public static String getSimpleName(Attribute<?> attribute)
+ {
+ return getSimpleAttributeName(attribute.name);
+ }
+
+ }
+
public class JSONObject2String implements StringCodec<Object>, Serializable
{
private static final long serialVersionUID = -664977453308585878L;
@@ -132,11 +908,11 @@ public class LogicalPlanConfiguration {
@Override
public Object fromString(String jsonObj)
{
-
+ LOG.debug("JONString {}", jsonObj);
ObjectMapper mapper = ObjectMapperFactory.getOperatorValueDeserializer();
try {
return mapper.readValue(jsonObj, Object.class);
- } catch (Exception e) {
+ } catch (IOException e) {
throw new RuntimeException("Error parsing json content", e);
}
}
@@ -147,13 +923,16 @@ public class LogicalPlanConfiguration {
ObjectMapper mapper = ObjectMapperFactory.getOperatorValueDeserializer();
try {
return mapper.writeValueAsString(pojo);
- } catch (Exception e) {
+ } catch (IOException e) {
throw new RuntimeException("Error writing object as json", e);
}
}
-
+
}
+ /**
+ * This is an abstract class representing the configuration applied to an element in the DAG.
+ */
private static abstract class Conf {
protected Conf parentConf = null;
@@ -184,7 +963,7 @@ public class LogicalPlanConfiguration {
@SuppressWarnings("unchecked")
public <T extends Conf> T getAncestorConf(StramElement ancestorElement) {
- if (getElement() == ancestorElement) {
+ if (getConfElement().getStramElement() == ancestorElement) {
return (T)this;
}
if (parentConf == null) {
@@ -221,7 +1000,7 @@ public class LogicalPlanConfiguration {
}
public <T extends Conf> List<T> getMatchingChildConf(String name, StramElement childType) {
- List<T> childConfs = new ArrayList<T>();
+ List<T> childConfs = new ArrayList<>();
Map<String, T> elChildren = getChildren(childType);
for (Map.Entry<String, T> entry : elChildren.entrySet()) {
String key = entry.getKey();
@@ -267,14 +1046,18 @@ public class LogicalPlanConfiguration {
conf = declaredConstructor.newInstance(new Object[] {});
conf.setId(id);
map.put(id, conf);
- } catch (Exception e) {
+ } catch (IllegalAccessException |
+ IllegalArgumentException |
+ InstantiationException |
+ NoSuchMethodException |
+ SecurityException |
+ InvocationTargetException e) {
LOG.error("Error instantiating configuration", e);
}
}
return conf;
}
- //public abstract Conf getChild(String id, StramElement childType);
public <T extends Conf> T getChild(String id, StramElement childType) {
T conf = null;
@SuppressWarnings("unchecked")
@@ -290,7 +1073,7 @@ public class LogicalPlanConfiguration {
// Always return non null so caller will not have to do extra check as expected
Map<String, T> elChildren = (Map<String, T>)children.get(childType);
if (elChildren == null) {
- elChildren = new HashMap<String, T>();
+ elChildren = Maps.newHashMap();
children.put(childType, elChildren);
}
return elChildren;
@@ -301,10 +1084,6 @@ public class LogicalPlanConfiguration {
public void parseElement(StramElement element, String[] keys, int index, String propertyValue) {
}
- public Class<? extends Context> getAttributeContextClass() {
- return null;
- }
-
public boolean isAllowedChild(StramElement childType) {
StramElement[] childElements = getChildElements();
if (childElements != null) {
@@ -318,7 +1097,7 @@ public class LogicalPlanConfiguration {
}
public StramElement getDefaultChildElement() {
- if ((getAttributeContextClass() == null) && isAllowedChild(StramElement.PROP)) {
+ if ((getConfElement().getContextClass() == null) && isAllowedChild(StramElement.PROP)) {
return StramElement.PROP;
}
return null;
@@ -330,8 +1109,7 @@ public class LogicalPlanConfiguration {
public abstract StramElement[] getChildElements();
- public abstract StramElement getElement();
-
+ public abstract ConfElement getConfElement();
}
private static class StramConf extends Conf {
@@ -345,21 +1123,20 @@ public class LogicalPlanConfiguration {
}
@Override
- public StramElement getElement()
+ public StramElement[] getChildElements()
{
- return null;
+ return CHILD_ELEMENTS;
}
@Override
- public StramElement[] getChildElements()
+ public ConfElement getConfElement()
{
- return CHILD_ELEMENTS;
+ return ConfElement.STRAM;
}
-
}
/**
- * App configuration
+ * This holds the configuration information for an Apex application.
*/
private static class AppConf extends Conf {
@@ -372,12 +1149,6 @@ public class LogicalPlanConfiguration {
}
@Override
- public StramElement getElement()
- {
- return StramElement.APPLICATION;
- }
-
- @Override
public void parseElement(StramElement element, String[] keys, int index, String propertyValue) {
if ((element == StramElement.CLASS) || (element == StramElement.PATH)) {
StramConf stramConf = getParentConf();
@@ -391,17 +1162,17 @@ public class LogicalPlanConfiguration {
return CHILD_ELEMENTS;
}
- @Override
- public Class<? extends Context> getAttributeContextClass()
- {
- return Context.DAGContext.class;
- }
@Override
public StramElement getDefaultChildElement() {
return StramElement.PROP;
}
+ @Override
+ public ConfElement getConfElement()
+ {
+ return ConfElement.APPLICATION;
+ }
}
private static class GatewayConf extends Conf {
@@ -419,11 +1190,10 @@ public class LogicalPlanConfiguration {
}
@Override
- public StramElement getElement()
+ public ConfElement getConfElement()
{
- return StramElement.GATEWAY;
+ return ConfElement.GATEWAY;
}
-
}
/**
@@ -444,11 +1214,10 @@ public class LogicalPlanConfiguration {
return CHILD_ELEMENTS;
}
-
@Override
- public StramElement getElement()
+ public ConfElement getConfElement()
{
- return StramElement.TEMPLATE;
+ return ConfElement.TEMPLATE;
}
@Override
@@ -472,23 +1241,23 @@ public class LogicalPlanConfiguration {
}
/**
- *
+ * This holds the configuration information for a stream that connects two operators in an Apex application.
*/
private static class StreamConf extends Conf {
private static final StramElement[] CHILD_ELEMENTS = new StramElement[] {StramElement.TEMPLATE, StramElement.PROP};
private OperatorConf sourceNode;
- private final Set<OperatorConf> targetNodes = new HashSet<OperatorConf>();
+ private final Set<OperatorConf> targetNodes = new HashSet<>();
@SuppressWarnings("unused")
StreamConf() {
}
@Override
- public StramElement getElement()
+ public ConfElement getConfElement()
{
- return StramElement.STREAM;
+ return ConfElement.STREAM;
}
/**
@@ -534,7 +1303,6 @@ public class LogicalPlanConfiguration {
throw new IllegalArgumentException("Duplicate " + name);
}
String[] parts = getNodeAndPortId(value);
- //setSource(parts[1], getOrAddOperator(appConf, parts[0]));
setSource(parts[1], appConf.getOrAddChild(parts[0], StramElement.OPERATOR, OperatorConf.class));
} else if (STREAM_SINKS.equals(name)) {
String[] targetPorts = value.split(",");
@@ -575,7 +1343,7 @@ public class LogicalPlanConfiguration {
}
/**
- *
+ * This is a simple extension of {@link java.util.Properties} which allows you to specify default properties.
*/
private static class PropertiesWithModifiableDefaults extends Properties {
private static final long serialVersionUID = -4675421720308249982L;
@@ -589,7 +1357,7 @@ public class LogicalPlanConfiguration {
}
/**
- * Operator configuration
+ * This holds the configuration information for an operator in an Apex application.
*/
private static class OperatorConf extends Conf {
@@ -599,14 +1367,14 @@ public class LogicalPlanConfiguration {
@SuppressWarnings("unused")
OperatorConf() {
}
- private final Map<String, StreamConf> inputs = new HashMap<String, StreamConf>();
- private final Map<String, StreamConf> outputs = new HashMap<String, StreamConf>();
+ private final Map<String, StreamConf> inputs = Maps.newHashMap();
+ private final Map<String, StreamConf> outputs = Maps.newHashMap();
private String templateRef;
@Override
- public StramElement getElement()
+ public ConfElement getConfElement()
{
- return StramElement.OPERATOR;
+ return ConfElement.OPERATOR;
}
@Override
@@ -651,12 +1419,6 @@ public class LogicalPlanConfiguration {
return StramElement.PROP;
}
- @Override
- public Class<? extends Context> getAttributeContextClass()
- {
- return OperatorContext.class;
- }
-
/**
*
* @return String
@@ -671,7 +1433,7 @@ public class LogicalPlanConfiguration {
}
/**
- * Port configuration
+ * This holds the configuration information for a port on an operator in an Apex application.
*/
private static class PortConf extends Conf {
@@ -682,23 +1444,16 @@ public class LogicalPlanConfiguration {
}
@Override
- public StramElement getElement()
- {
- return StramElement.PORT;
- }
-
- @Override
public StramElement[] getChildElements()
{
return CHILD_ELEMENTS;
}
@Override
- public Class<? extends Context> getAttributeContextClass()
+ public ConfElement getConfElement()
{
- return PortContext.class;
+ return ConfElement.PORT;
}
-
}
private static final Map<StramElement, Class<? extends Conf>> elementMaps = Maps.newHashMap();
@@ -715,8 +1470,8 @@ public class LogicalPlanConfiguration {
elementMaps.put(StramElement.OUTPUT_PORT, PortConf.class);
}
- private Conf getConf(StramElement element, Conf ancestorConf) {
- if (element == ancestorConf.getElement()) {
+ private static Conf getConf(StramElement element, Conf ancestorConf) {
+ if (element == ancestorConf.getConfElement().getStramElement()) {
return ancestorConf;
}
// If top most element is reached and didnt match ancestor conf
@@ -724,13 +1479,13 @@ public class LogicalPlanConfiguration {
if (element == null) {
return null;
}
- StramElement parentElement = getAllowedParentElement(element, ancestorConf);
+ StramElement parentElement = ConfElement.getAllowedParentConf(element);
Conf parentConf = getConf(parentElement, ancestorConf);
return parentConf.getOrAddChild(WILDCARD, element, elementMaps.get(element));
}
- private Conf addConf(StramElement element, String name, Conf ancestorConf) {
- StramElement parentElement = getAllowedParentElement(element, ancestorConf);
+ private static Conf addConf(StramElement element, String name, Conf ancestorConf) {
+ StramElement parentElement = ConfElement.getAllowedParentConf(element);
Conf conf1 = null;
Conf parentConf = getConf(parentElement, ancestorConf);
if (parentConf != null) {
@@ -739,28 +1494,8 @@ public class LogicalPlanConfiguration {
return conf1;
}
- private StramElement getAllowedParentElement(StramElement element, Conf ancestorConf) {
- StramElement parentElement = null;
- if ((element == StramElement.APPLICATION)) {
- parentElement = null;
- } else if ((element == StramElement.GATEWAY) || (element == StramElement.OPERATOR) || (element == StramElement.STREAM)) {
- parentElement = StramElement.APPLICATION;
- } else if ((element == StramElement.PORT) || (element == StramElement.INPUT_PORT) || (element == StramElement.OUTPUT_PORT)) {
- parentElement = StramElement.OPERATOR;
- } else if (element == StramElement.TEMPLATE) {
- parentElement = null;
- }
- return parentElement;
- }
-
- /*
- private boolean isApplicationTypeConf(Conf conf) {
- return (conf.getElement() == null) || (conf.getElement() == StramElement.APPLICATION);
- }
- */
-
private <T extends Conf> List<T> getMatchingChildConf(List<? extends Conf> confs, String name, StramElement childType) {
- List<T> childConfs = new ArrayList<T>();
+ List<T> childConfs = Lists.newArrayList();
for (Conf conf1 : confs) {
List<T> matchingConfs = conf1.getMatchingChildConf(name, childType);
childConfs.addAll(matchingConfs);
@@ -813,7 +1548,7 @@ public class LogicalPlanConfiguration {
public String getAppAlias(String appPath) {
String appAlias;
if (appPath.endsWith(CLASS_SUFFIX)) {
- appPath = appPath.replace("/", ".").substring(0, appPath.length() - CLASS_SUFFIX.length());
+ appPath = appPath.replace("/", KEY_SEPARATOR).substring(0, appPath.length() - CLASS_SUFFIX.length());
}
appAlias = stramConf.appAliases.get(appPath);
if (appAlias == null) {
@@ -836,11 +1571,11 @@ public class LogicalPlanConfiguration {
JSONArray operatorArray = json.getJSONArray("operators");
for (int i = 0; i < operatorArray.length(); i++) {
JSONObject operator = operatorArray.getJSONObject(i);
- String operatorPrefix = StreamingApplication.DT_PREFIX + StramElement.OPERATOR.getValue() + "." + operator.getString("name") + ".";
+ String operatorPrefix = StreamingApplication.DT_PREFIX + StramElement.OPERATOR.getValue() + KEY_SEPARATOR + operator.getString("name") + ".";
prop.setProperty(operatorPrefix + "classname", operator.getString("class"));
JSONObject operatorProperties = operator.optJSONObject("properties");
if (operatorProperties != null) {
- String propertiesPrefix = operatorPrefix + StramElement.PROP.getValue() + ".";
+ String propertiesPrefix = operatorPrefix + StramElement.PROP.getValue() + KEY_SEPARATOR;
@SuppressWarnings("unchecked")
Iterator<String> iter = operatorProperties.keys();
while (iter.hasNext()) {
@@ -850,7 +1585,7 @@ public class LogicalPlanConfiguration {
}
JSONObject operatorAttributes = operator.optJSONObject("attributes");
if (operatorAttributes != null) {
- String attributesPrefix = operatorPrefix + StramElement.ATTR.getValue() + ".";
+ String attributesPrefix = operatorPrefix + StramElement.ATTR.getValue() + KEY_SEPARATOR;
@SuppressWarnings("unchecked")
Iterator<String> iter = operatorAttributes.keys();
while (iter.hasNext()) {
@@ -860,12 +1595,12 @@ public class LogicalPlanConfiguration {
}
JSONArray portArray = operator.optJSONArray("ports");
if (portArray != null) {
- String portsPrefix = operatorPrefix + StramElement.PORT.getValue() + ".";
+ String portsPrefix = operatorPrefix + StramElement.PORT.getValue() + KEY_SEPARATOR;
for (int j = 0; j < portArray.length(); j++) {
JSONObject port = portArray.getJSONObject(j);
JSONObject portAttributes = port.optJSONObject("attributes");
if (portAttributes != null) {
- String portAttributePrefix = portsPrefix + port.getString("name") + "." + StramElement.ATTR.getValue() + ".";
+ String portAttributePrefix = portsPrefix + port.getString("name") + KEY_SEPARATOR + StramElement.ATTR.getValue() + KEY_SEPARATOR;
@SuppressWarnings("unchecked")
Iterator<String> iter = portAttributes.keys();
while (iter.hasNext()) {
@@ -876,10 +1611,10 @@ public class LogicalPlanConfiguration {
}
}
}
-
+
JSONObject appAttributes = json.optJSONObject("attributes");
if (appAttributes != null) {
- String attributesPrefix = StreamingApplication.DT_PREFIX + StramElement.ATTR.getValue() + ".";
+ String attributesPrefix = StreamingApplication.DT_PREFIX + StramElement.ATTR.getValue() + KEY_SEPARATOR;
@SuppressWarnings("unchecked")
Iterator<String> iter = appAttributes.keys();
while (iter.hasNext()) {
@@ -892,9 +1627,9 @@ public class LogicalPlanConfiguration {
for (int i = 0; i < streamArray.length(); i++) {
JSONObject stream = streamArray.getJSONObject(i);
String name = stream.optString("name", "stream-" + i);
- String streamPrefix = StreamingApplication.DT_PREFIX + StramElement.STREAM.getValue() + "." + name + ".";
+ String streamPrefix = StreamingApplication.DT_PREFIX + StramElement.STREAM.getValue() + KEY_SEPARATOR + name + KEY_SEPARATOR;
JSONObject source = stream.getJSONObject("source");
- prop.setProperty(streamPrefix + STREAM_SOURCE, source.getString("operatorName") + "." + source.getString("portName"));
+ prop.setProperty(streamPrefix + STREAM_SOURCE, source.getString("operatorName") + KEY_SEPARATOR + source.getString("portName"));
JSONArray sinks = stream.getJSONArray("sinks");
StringBuilder sinkPropertyValue = new StringBuilder();
for (int j = 0; j < sinks.length(); j++) {
@@ -902,7 +1637,7 @@ public class LogicalPlanConfiguration {
sinkPropertyValue.append(",");
}
JSONObject sink = sinks.getJSONObject(j);
- sinkPropertyValue.append(sink.getString("operatorName")).append(".").append(sink.getString("portName"));
+ sinkPropertyValue.append(sink.getString("operatorName")).append(KEY_SEPARATOR).append(sink.getString("portName"));
}
prop.setProperty(streamPrefix + STREAM_SINKS, sinkPropertyValue.toString());
String locality = stream.optString("locality", null);
@@ -937,13 +1672,21 @@ public class LogicalPlanConfiguration {
String propertyValue = props.getProperty(propertyName);
this.properties.setProperty(propertyName, propertyValue);
if (propertyName.startsWith(StreamingApplication.DT_PREFIX)) {
- String[] keyComps = propertyName.split("\\.");
+ String[] keyComps = propertyName.split(KEY_SEPARATOR_SPLIT_REGEX);
parseStramPropertyTokens(keyComps, 1, propertyName, propertyValue, stramConf);
}
}
return this;
}
+ /**
+ * This method is used to parse an Apex property name.
+ * @param keys The keys into which an Apex property is split into.
+ * @param index The current index that the parser is on for processing the property name.
+ * @param propertyName The original unsplit Apex property name.
+ * @param propertyValue The value corresponding to the Apex property.
+ * @param conf
+ */
private void parseStramPropertyTokens(String[] keys, int index, String propertyName, String propertyValue, Conf conf) {
if (index < keys.length) {
String key = keys[index];
@@ -973,14 +1716,65 @@ public class LogicalPlanConfiguration {
LOG.error("Invalid configuration key: {}", propertyName);
}
} else if ((element == StramElement.ATTR) || ((element == null) && (conf.getDefaultChildElement() == StramElement.ATTR))) {
- if (conf.getElement() == null) {
+ String attributeName = AttributeParseUtils.getAttributeName(element, keys, index);
+
+ if (element != StramElement.ATTR) {
+ String expName = getCompleteKey(keys, 0, index) + KEY_SEPARATOR + StramElement.ATTR.getValue() + KEY_SEPARATOR + attributeName;
+ LOG.warn("Referencing the attribute as {} instead of {} is deprecated!", getCompleteKey(keys, 0), expName);
+ }
+
+ if (conf.getConfElement().getStramElement() == null) {
conf = addConf(StramElement.APPLICATION, WILDCARD, conf);
}
+
if (conf != null) {
- // Supporting current implementation where attribute can be directly specified under stram
- // Re-composing complete key for nested keys which are used in templates
- // Implement it better way to not pre-tokenize the property string and parse progressively
- parseAttribute(conf, keys, index, element, propertyValue);
+ if (AttributeParseUtils.isSimpleAttributeName(attributeName)) {
+ //The provided attribute name was a simple name
+
+ if (!AttributeParseUtils.ALL_SIMPLE_ATTRIBUTE_NAMES.contains(attributeName)) {
+ throw new ValidationException("Invalid attribute reference: " + getCompleteKey(keys, 0));
+ }
+
+ if (!conf.getConfElement().getAllChildAttributes().contains(attributeName)) {
+ throw new ValidationException(attributeName
+ + " is not defined for the "
+ + conf.getConfElement().getStramElement()
+ + " or any of its child configurations.");
+ }
+
+ if (conf.getConfElement().getAmbiguousAttributes().contains(attributeName)) {
+ //If the attribute name is ambiguous at this configuration level we should tell the user.
+ LOG.warn("The attribute "
+ + attributeName
+ + " is ambiguous when specified on an " + conf.getConfElement().getStramElement());
+ }
+
+ if (conf.getConfElement().getContextAttributes().contains(attributeName)) {
+ @SuppressWarnings("unchecked")
+ Attribute<Object> attr = (Attribute<Object>)ContextUtils.CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(conf.getConfElement().getContextClass()).get(attributeName);
+ conf.setAttribute(attr, propertyValue);
+ } else {
+ AttributeParseUtils.processAllConfsForAttribute(conf, attributeName, propertyValue);
+ }
+ } else {
+ //This is a FQ attribute name
+ Class<? extends Context> contextClass = AttributeParseUtils.getContainingContextClass(attributeName);
+
+ //Convert to a simple name
+ attributeName = AttributeParseUtils.getSimpleAttributeName(attributeName);
+
+ if (!ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass).contains(attributeName)) {
+ throw new ValidationException(attributeName + " is not a valid attribute in " + contextClass.getCanonicalName());
+ }
+
+ ConfElement confWithAttr = ConfElement.CONTEXT_TO_CONF_ELEMENT.get(contextClass);
+
+ conf = ConfElement.addConfs(conf, confWithAttr);
+
+ @SuppressWarnings("unchecked")
+ Attribute<Object> attr = (Attribute<Object>)ContextUtils.CONTEXT_TO_ATTRIBUTE_NAME_TO_ATTRIBUTE.get(confWithAttr.getContextClass()).get(attributeName);
+ conf.setAttribute(attr, propertyValue);
+ }
} else {
LOG.error("Invalid configuration key: {}", propertyName);
}
@@ -992,12 +1786,6 @@ public class LogicalPlanConfiguration {
prop = getCompleteKey(keys, index+1);
} else {
prop = getCompleteKey(keys, index);
- /*
- if (conf.getAttributeContextClass() != null) {
- LOG.warn("Please specify the property {} using the {} keyword as {}", prop, StramElement.PROP.getValue(),
- getCompleteKey(keys, 0, index) + "." + StramElement.PROP.getValue() + "." + getCompleteKey(keys, index));
- }
- */
}
if (prop != null) {
conf.setProperty(prop, propertyValue);
@@ -1023,15 +1811,30 @@ public class LogicalPlanConfiguration {
return element;
}
- private String getCompleteKey(String[] keys, int start) {
+ /**
+ * This constructs a string from the keys in the given keys array starting from
+ * the start index inclusive until the end of the array.
+ * @param keys The keys from which to construct a string.
+ * @param start The token to start creating a string from.
+ * @return The completed key.
+ */
+ private static String getCompleteKey(String[] keys, int start) {
return getCompleteKey(keys, start, keys.length);
}
- private String getCompleteKey(String[] keys, int start, int end) {
+ /**
+ * This constructs a string from the keys in the given keys array starting from
+ * the start index inclusive until the specified end index exclusive.
+ * @param keys The keys from which to construct a string.
+ * @param start The token to start creating a string from.
+ * @param end 1 + the last index to include in the concatenation.
+ * @return The completed key.
+ */
+ private static String getCompleteKey(String[] keys, int start, int end) {
StringBuilder sb = new StringBuilder(1024);
for (int i = start; i < end; ++i) {
if (i > start) {
- sb.append(".");
+ sb.append(KEY_SEPARATOR);
}
sb.append(keys[i]);
}
@@ -1099,7 +1902,7 @@ public class LogicalPlanConfiguration {
Map<String, OperatorConf> operators = appConf.getChildren(StramElement.OPERATOR);
- Map<OperatorConf, Operator> nodeMap = new HashMap<OperatorConf, Operator>(operators.size());
+ Map<OperatorConf, Operator> nodeMap = Maps.newHashMapWithExpectedSize(operators.size());
// add all operators first
for (Map.Entry<String, OperatorConf> nodeConfEntry : operators.entrySet()) {
OperatorConf nodeConf = nodeConfEntry.getValue();
@@ -1117,7 +1920,7 @@ public class LogicalPlanConfiguration {
nd = dag.addOperator(nodeConfEntry.getKey(), nodeClass);
}
setOperatorProperties(nd, nodeConf.getProperties());
- } catch (Exception e) {
+ } catch (IOException e) {
throw new IllegalArgumentException("Error setting operator properties " + e.getMessage(), e);
}
nodeMap.put(nodeConf, nd);
@@ -1178,9 +1981,9 @@ public class LogicalPlanConfiguration {
/**
* Populate the logical plan from the streaming application definition and configuration.
* Configuration is resolved based on application alias, if any.
- * @param app
- * @param dag
- * @param name
+ * @param app The {@lin StreamingApplication} to be run.
+ * @param dag This will hold the {@link LogicalPlan} representation of the given {@link StreamingApplication}.
+ * @param name The path of the application class in the jar.
*/
public void prepareDAG(LogicalPlan dag, StreamingApplication app, String name)
{
@@ -1211,10 +2014,6 @@ public class LogicalPlanConfiguration {
return props;
}
- private String getSimpleName(Attribute<?> attribute) {
- return attribute.name.substring(attribute.name.lastIndexOf('.')+1);
- }
-
/**
* Get the configuration opProps for the given operator.
* These can be operator specific settings or settings from matching templates.
@@ -1229,7 +2028,7 @@ public class LogicalPlanConfiguration {
}
private Map<String,String> getApplicationProperties(List<AppConf> appConfs){
- Map<String, String> appProps = new HashMap<String, String>();
+ Map<String, String> appProps = Maps.newHashMap();
// Apply the configurations in reverse order since the higher priority ones are at the beginning
for(int i = appConfs.size()-1; i >= 0; i--){
AppConf conf1 = appConfs.get(i);
@@ -1246,7 +2045,7 @@ public class LogicalPlanConfiguration {
*/
private Map<String, String> getProperties(OperatorMeta ow, List<OperatorConf> opConfs, String appName)
{
- Map<String, String> opProps = new HashMap<String, String>();
+ Map<String, String> opProps = Maps.newHashMap();
Map<String, TemplateConf> templates = stramConf.getChildren(StramElement.TEMPLATE);
// list of all templates that match operator, ordered by priority
if (!templates.isEmpty()) {
@@ -1269,12 +2068,11 @@ public class LogicalPlanConfiguration {
Conf conf1 = opConfs.get(i);
opProps.putAll(Maps.fromProperties(conf1.properties));
}
- //properties.remove(OPERATOR_CLASSNAME);
return opProps;
}
private List<TemplateConf> getDirectTemplates(List<OperatorConf> opConfs, Map<String, TemplateConf> templates) {
- List<TemplateConf> refTemplates = new ArrayList<TemplateConf>();
+ List<TemplateConf> refTemplates = Lists.newArrayList();
for (TemplateConf t : templates.values()) {
for (OperatorConf opConf : opConfs) {
if (t.id.equals(opConf.templateRef)) {
@@ -1293,13 +2091,8 @@ public class LogicalPlanConfiguration {
* @return TreeMap<Integer, TemplateConf>
*/
private TreeMap<Integer, TemplateConf> getMatchingTemplates(OperatorMeta ow, String appName, Map<String, TemplateConf> templates) {
- TreeMap<Integer, TemplateConf> tm = new TreeMap<Integer, TemplateConf>();
+ TreeMap<Integer, TemplateConf> tm = Maps.newTreeMap();
for (TemplateConf t : templates.values()) {
- /*if (t.id == nodeConf.templateRef) {
- // directly assigned applies last
- tm.put(1, t);
- continue;
- } else*/
if ((t.idRegExp != null && ow.getName().matches(t.idRegExp))) {
tm.put(1, t);
} else if (appName != null && t.appNameRegExp != null
@@ -1326,10 +2119,7 @@ public class LogicalPlanConfiguration {
BeanUtils.populate(operator, properties);
return operator;
}
- catch (IllegalAccessException e) {
- throw new IllegalArgumentException("Error setting operator properties", e);
- }
- catch (InvocationTargetException e) {
+ catch (IllegalAccessException | InvocationTargetException e) {
throw new IllegalArgumentException("Error setting operator properties", e);
}
}
@@ -1340,10 +2130,7 @@ public class LogicalPlanConfiguration {
BeanUtils.populate(application, properties);
return application;
}
- catch (IllegalAccessException e) {
- throw new IllegalArgumentException("Error setting application properties", e);
- }
- catch (InvocationTargetException e) {
+ catch (IllegalAccessException | InvocationTargetException e) {
throw new IllegalArgumentException("Error setting application properties", e);
}
}
@@ -1369,21 +2156,7 @@ public class LogicalPlanConfiguration {
setOperatorProperties(ow.getOperator(), opProps);
}
}
-/*
- private static final Map<String, Attribute<?>> legacyKeyMap = Maps.newHashMap();
- static {
- legacyKeyMap.put("appName", Context.DAGContext.APPLICATION_NAME);
- legacyKeyMap.put("libjars", Context.DAGContext.LIBRARY_JARS);
- legacyKeyMap.put("maxContainers", Context.DAGContext.CONTAINERS_MAX_COUNT);
- legacyKeyMap.put("containerMemoryMB", Context.DAGContext.CONTAINER_MEMORY_MB);
- legacyKeyMap.put("containerJvmOpts", Context.DAGContext.CONTAINER_JVM_OPTIONS);
- legacyKeyMap.put("masterMemoryMB", Context.DAGContext.MASTER_MEMORY_MB);
- legacyKeyMap.put("windowSizeMillis", Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
- legacyKeyMap.put("appPath", Context.DAGContext.APPLICATION_PATH);
- legacyKeyMap.put("allocateResourceTimeoutMillis", Context.DAGContext.RESOURCE_ALLOCATION_TIMEOUT_MILLIS);
- }
-*/
/**
* Set the application configuration.
* @param dag
@@ -1397,18 +2170,7 @@ public class LogicalPlanConfiguration {
}
private void setApplicationConfiguration(final LogicalPlan dag, List<AppConf> appConfs,StreamingApplication app) {
- // Make the gateway address available as an application attribute
-// for (Conf appConf : appConfs) {
-// Conf gwConf = appConf.getChild(null, StramElement.GATEWAY);
-// if (gwConf != null) {
-// String gatewayAddress = gwConf.properties.getProperty(GATEWAY_LISTEN_ADDRESS_PROP);
-// if (gatewayAddress != null) {
-// dag.setAttribute(DAGContext.GATEWAY_CONNECT_ADDRESS, gatewayAddress);
-// break;
-// }
-// }
-// }
- setAttributes(Context.DAGContext.class, appConfs, dag.getAttributes());
+ setAttributes(appConfs, dag.getAttributes());
if (app != null) {
Map<String, String> appProps = getApplicationProperties(appConfs);
setApplicationProperties(app, appProps);
@@ -1420,7 +2182,7 @@ public class LogicalPlanConfiguration {
List<OperatorConf> opConfs = getMatchingChildConf(appConfs, ow.getName(), StramElement.OPERATOR);
// Set the operator attributes
- setAttributes(OperatorContext.class, opConfs, ow.getAttributes());
+ setAttributes(opConfs, ow.getAttributes());
// Set the operator opProps
Map<String, String> opProps = getProperties(ow, opConfs, appName);
setOperatorProperties(ow.getOperator(), opProps);
@@ -1432,7 +2194,7 @@ public class LogicalPlanConfiguration {
// Add the generic port attributes as well
List<PortConf> portConfs = getMatchingChildConf(opConfs, im.getPortName(), StramElement.PORT);
inPortConfs.addAll(portConfs);
- setAttributes(PortContext.class, inPortConfs, im.getAttributes());
+ setAttributes(inPortConfs, im.getAttributes());
}
for (Entry<LogicalPlan.OutputPortMeta, LogicalPlan.StreamMeta> entry : ow.getOutputStreams().entrySet()) {
@@ -1441,7 +2203,7 @@ public class LogicalPlanConfiguration {
// Add the generic port attributes as well
List<PortConf> portConfs = getMatchingChildConf(opConfs, om.getPortName(), StramElement.PORT);
outPortConfs.addAll(portConfs);
- setAttributes(PortContext.class, outPortConfs, om.getAttributes());
+ setAttributes(outPortConfs, om.getAttributes());
}
ow.populateAggregatorMeta();
}
@@ -1460,36 +2222,7 @@ public class LogicalPlanConfiguration {
}
}
- private final Map<Class<? extends Context>, Map<String, Attribute<Object>>> attributeMap = Maps.newHashMap();
-
- private void parseAttribute(Conf conf, String[] keys, int index, StramElement element, String attrValue)
- {
- String configKey = (element == StramElement.ATTR) ? getCompleteKey(keys, index + 1) : getCompleteKey(keys, index);
- boolean isDeprecated = false;
- Class<? extends Context> clazz = conf.getAttributeContextClass();
- Map<String, Attribute<Object>> m = attributeMap.get(clazz);
- if (!attributeMap.containsKey(clazz)) {
- Set<Attribute<Object>> attributes = AttributeInitializer.getAttributes(clazz);
- m = Maps.newHashMapWithExpectedSize(attributes.size());
- for (Attribute<Object> attr : attributes) {
- m.put(getSimpleName(attr), attr);
- }
- attributeMap.put(clazz, m);
- }
- Attribute<Object> attr = m.get(configKey);
- if (attr == null) {
- throw new ValidationException("Invalid attribute reference: " + getCompleteKey(keys, 0));
- }
-
- if (element != StramElement.ATTR || isDeprecated) {
- String expName = getCompleteKey(keys, 0, index) + "." + StramElement.ATTR.getValue() + "." + getSimpleName(attr);
- LOG.warn("Referencing the attribute as {} instead of {} is deprecated!", getCompleteKey(keys, 0), expName);
- }
-
- conf.setAttribute(attr, attrValue);
- }
-
- private void setAttributes(Class<?> clazz, List<? extends Conf> confs, Attribute.AttributeMap attributeMap) {
+ private void setAttributes(List<? extends Conf> confs, Attribute.AttributeMap attributeMap) {
Set<Attribute<Object>> processedAttributes = Sets.newHashSet();
//json object codec for complex attributes
JSONObject2String jsonCodec = new JSONObject2String();