You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2017/10/01 15:49:29 UTC
[apex-malhar] 01/02: APEXMALHAR-2542-Fix test failures by altering
the location property
This is an automated email from the ASF dual-hosted git repository.
thw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apex-malhar.git
commit d8ec1736238dbd140f3b63458930f2260b5a8ca2
Author: Ananth <an...@gmail.com>
AuthorDate: Sat Sep 30 03:53:33 2017 +1000
APEXMALHAR-2542-Fix test failures by altering the location property
---
.../apex/examples/enricher/DataGenerator.java | 27 +++---
.../examples/enricher/EnricherAppWithJSONFile.java | 23 +++++-
.../apex/examples/enricher/POJOEnriched.java | 14 ++--
.../src/main/resources/META-INF/properties.xml | 8 +-
.../apex/examples/enricher/ApplicationTest.java | 96 +++++++++++++++++++---
5 files changed, 128 insertions(+), 40 deletions(-)
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
index b10d12c..12ffc81 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
@@ -40,8 +40,8 @@ public class DataGenerator extends BaseOperator implements InputOperator
public static int NUM_CIRCLES = 10;
private Random r;
- private int count = 0;
- private int limit = 1000;
+ private volatile int count = 0;
+ private int limit = 15;
public final transient DefaultOutputPort<byte[]> output = new DefaultOutputPort<>();
@@ -52,17 +52,11 @@ public class DataGenerator extends BaseOperator implements InputOperator
}
@Override
- public void beginWindow(long windowId)
- {
- super.beginWindow(windowId);
- count = 0;
- }
-
- @Override
public void emitTuples()
{
- if (count++ < limit) {
+ if (count < limit) {
output.emit(getRecord());
+ count++;
}
}
@@ -71,8 +65,7 @@ public class DataGenerator extends BaseOperator implements InputOperator
String phone = getRandomNumber(10);
String imsi = getHashInRange(phone, 15);
String imei = getHashInRange(imsi, 15);
- String circleId = Math.abs(phone.hashCode()) % NUM_CIRCLES + "";
-// String record = MessageFormat.format(baseDataTemplate, phone, imsi, imei, circleId);
+ String circleId = Math.abs(count) % NUM_CIRCLES + "";
String record = "{" + "\"phone\":\"" + phone + "\","
+ "\"imei\":\"" + imei + "\","
+ "\"imsi\":\"" + imsi + "\","
@@ -111,4 +104,14 @@ public class DataGenerator extends BaseOperator implements InputOperator
{
this.limit = limit;
}
+
+ public int getCount()
+ {
+ return count;
+ }
+
+ public void setCount(int count)
+ {
+ this.count = count;
+ }
}
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
index 10024c8..25330e2 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
@@ -29,7 +29,6 @@ import com.datatorrent.api.annotation.ApplicationAnnotation;
import com.datatorrent.contrib.enrich.JsonFSLoader;
import com.datatorrent.contrib.enrich.POJOEnricher;
import com.datatorrent.contrib.parser.JsonParser;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
@ApplicationAnnotation(name = "EnricherAppWithJSONFile")
/**
@@ -37,10 +36,15 @@ import com.datatorrent.lib.io.ConsoleOutputOperator;
*/
public class EnricherAppWithJSONFile implements StreamingApplication
{
+
+
+
+ DataGenerator dataGenerator;
+
@Override
public void populateDAG(DAG dag, Configuration conf)
{
- DataGenerator dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class);
+ dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class);
JsonParser parser = dag.addOperator("Parser", JsonParser.class);
/**
@@ -60,10 +64,21 @@ public class EnricherAppWithJSONFile implements StreamingApplication
enrich.setIncludeFields(includeFields);
enrich.setLookupFields(lookupFields);
- ConsoleOutputOperator console = dag.addOperator("Console", ConsoleOutputOperator.class);
+ LineOutputOperator fileOut = dag.addOperator("fileOut", LineOutputOperator.class);
+
dag.addStream("Parse", dataGenerator.output, parser.in);
dag.addStream("Enrich", parser.out, enrich.input);
- dag.addStream("Console", enrich.output, console.input);
+ dag.addStream("Console", enrich.output, fileOut.input);
+ }
+
+ public DataGenerator getDataGenerator()
+ {
+ return dataGenerator;
+ }
+
+ public void setDataGenerator(DataGenerator dataGenerator)
+ {
+ this.dataGenerator = dataGenerator;
}
}
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
index d89986a..4b0ad45 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
@@ -83,12 +83,12 @@ public class POJOEnriched
@Override
public String toString()
{
- return "POJOEnriched{" +
- "phone='" + phone + '\'' +
- ", imei='" + imei + '\'' +
- ", imsi='" + imsi + '\'' +
- ", circleId=" + circleId +
- ", circleName='" + circleName + '\'' +
- '}';
+ return "{" +
+ "\"phone\":\"" + phone + "\"," +
+ "\"imei\":\"" + imei + "\"," +
+ "\"imsi\":\"" + imsi + "\"," +
+ "\"circleId\":" + circleId + "," +
+ "\"circleName\":\"" + circleName + "\"" +
+ "}";
}
}
diff --git a/examples/enricher/src/main/resources/META-INF/properties.xml b/examples/enricher/src/main/resources/META-INF/properties.xml
index c37be45..db581da 100644
--- a/examples/enricher/src/main/resources/META-INF/properties.xml
+++ b/examples/enricher/src/main/resources/META-INF/properties.xml
@@ -37,13 +37,13 @@
</property>
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.Enrich.prop.store.fileName</name>
- <value>/tmp/circleMapping.txt</value>
+ <value>./src/main/resources/circleMapping.txt</value>
</property>
<!-- file output operator -->
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.filePath</name>
- <value>/tmp/enrichedData</value>
+ <value>./target/enrichedData</value>
</property>
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.baseName</name>
@@ -51,11 +51,11 @@
</property>
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.maxLength</name>
- <value>1024</value>
+ <value>102400</value>
</property>
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.rotationWindows</name>
- <value>4</value>
+ <value>0</value>
</property>
</configuration>
diff --git a/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
index 6cbf846..8f1aed3 100644
--- a/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
+++ b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
@@ -18,14 +18,23 @@
*/
package org.apache.apex.examples.enricher;
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Test;
+import org.apache.apex.api.EmbeddedAppLauncher;
+import org.apache.apex.api.Launcher;
import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
+import com.datatorrent.api.Attribute;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
/**
* Test the DAG declaration in local mode.
@@ -36,16 +45,77 @@ public class ApplicationTest
@Test
public void testApplication() throws Exception
{
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new EnricherAppWithJSONFile(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000); // runs for 10 seconds and quits
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
+ Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
+ launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
+ EnricherAppWithJSONFile enricherAppWithJSONFile = new EnricherAppWithJSONFile();
+ Launcher.AppHandle appHandle = launcher.launchApp(enricherAppWithJSONFile, conf, launchAttributes);
+ File folder = new File("target/enrichedData");
+ File[] listOfFiles = null;
+ int sleepTimeCounterForLoopExit = 0;
+ int sleepTimePerIteration = 5000;
+ // First ensure that the output file is generated and the expected lines match
+ // The test main thread sleeps until the expected lines are generated in the output file
+ while (true) {
+ sleepTimeCounterForLoopExit += sleepTimePerIteration;
+ // break the loop of the test output is not generated by 40 seconds
+ if (sleepTimeCounterForLoopExit > 40000) {
+ break;
+ }
+ Thread.sleep(sleepTimePerIteration);
+ listOfFiles = folder.listFiles();
+ if ( listOfFiles.length == 0) {
+ continue;
+ }
+ String lineRead = null;
+ int counter = 0;
+ try (BufferedReader reader = new BufferedReader(new FileReader(listOfFiles[0].getAbsolutePath()))) {
+ while ((lineRead = reader.readLine()) != null) {
+ counter += 1;
+ }
+ }
+ if (counter < enricherAppWithJSONFile.getDataGenerator().getLimit()) {
+ continue;
+ } else {
+ break;
+ }
}
+ listOfFiles = folder.listFiles();
+ assertEnrichedOutput(listOfFiles);
}
+ private void assertEnrichedOutput(File[] listOfFiles) throws IOException
+ {
+ // Ensure that only one file is generated as output per properties configuration
+ assertEquals(1, listOfFiles.length);
+ String aJSONEntry = null;
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Integer> countsOfEnrichedValues = new HashMap<>();
+ String enrichedValue = null;
+ try (BufferedReader reader = new BufferedReader(new FileReader(listOfFiles[0].getAbsolutePath()))) {
+ while ((aJSONEntry = reader.readLine()) != null) {
+ POJOEnriched anEnrichedEntry = mapper.readValue(aJSONEntry, POJOEnriched.class);
+ enrichedValue = anEnrichedEntry.getCircleName();
+ assertNotNull(enrichedValue); // all values should have an entiched value
+ if (!countsOfEnrichedValues.containsKey(enrichedValue)) {
+ countsOfEnrichedValues.put(enrichedValue,1);
+ } else {
+ Integer currentCountForEnrichedEntry = countsOfEnrichedValues.get(enrichedValue);
+ countsOfEnrichedValues.put(enrichedValue, currentCountForEnrichedEntry + 1);
+ }
+ }
+ }
+ for (String circleName : countsOfEnrichedValues.keySet() ) {
+ int currentCircleNameAsciiCode = ((int)circleName.charAt(0));
+ Integer circleNameCount = countsOfEnrichedValues.get(circleName);
+ // All circle names "A" to "E" should have a count of 2 and all others should have 1
+ if ( ( currentCircleNameAsciiCode > 64 ) && (currentCircleNameAsciiCode < 70) ) {
+ assertEquals(2,circleNameCount.intValue());
+ } else {
+ assertEquals(1, circleNameCount.intValue());
+ }
+ }
+ }
}
--
To stop receiving notification emails like this one, please contact
"commits@apex.apache.org" <co...@apex.apache.org>.