You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2017/10/01 15:49:29 UTC

[apex-malhar] 01/02: APEXMALHAR-2542-Fix test failures by altering the location property

This is an automated email from the ASF dual-hosted git repository.

thw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apex-malhar.git

commit d8ec1736238dbd140f3b63458930f2260b5a8ca2
Author: Ananth <an...@gmail.com>
AuthorDate: Sat Sep 30 03:53:33 2017 +1000

    APEXMALHAR-2542-Fix test failures by altering the location property
---
 .../apex/examples/enricher/DataGenerator.java      | 27 +++---
 .../examples/enricher/EnricherAppWithJSONFile.java | 23 +++++-
 .../apex/examples/enricher/POJOEnriched.java       | 14 ++--
 .../src/main/resources/META-INF/properties.xml     |  8 +-
 .../apex/examples/enricher/ApplicationTest.java    | 96 +++++++++++++++++++---
 5 files changed, 128 insertions(+), 40 deletions(-)

diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
index b10d12c..12ffc81 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
@@ -40,8 +40,8 @@ public class DataGenerator extends BaseOperator implements InputOperator
   public static int NUM_CIRCLES = 10;
 
   private Random r;
-  private int count = 0;
-  private int limit = 1000;
+  private volatile int count = 0;
+  private int limit = 15;
 
   public final transient DefaultOutputPort<byte[]> output = new DefaultOutputPort<>();
 
@@ -52,17 +52,11 @@ public class DataGenerator extends BaseOperator implements InputOperator
   }
 
   @Override
-  public void beginWindow(long windowId)
-  {
-    super.beginWindow(windowId);
-    count = 0;
-  }
-
-  @Override
   public void emitTuples()
   {
-    if (count++ < limit) {
+    if (count < limit) {
       output.emit(getRecord());
+      count++;
     }
   }
 
@@ -71,8 +65,7 @@ public class DataGenerator extends BaseOperator implements InputOperator
     String phone = getRandomNumber(10);
     String imsi = getHashInRange(phone, 15);
     String imei = getHashInRange(imsi, 15);
-    String circleId = Math.abs(phone.hashCode()) % NUM_CIRCLES + "";
-//    String record = MessageFormat.format(baseDataTemplate, phone, imsi, imei, circleId);
+    String circleId = Math.abs(count) % NUM_CIRCLES + "";
     String record = "{" + "\"phone\":\"" + phone + "\","
         + "\"imei\":\"" + imei + "\","
         + "\"imsi\":\"" + imsi + "\","
@@ -111,4 +104,14 @@ public class DataGenerator extends BaseOperator implements InputOperator
   {
     this.limit = limit;
   }
+
+  public int getCount()
+  {
+    return count;
+  }
+
+  public void setCount(int count)
+  {
+    this.count = count;
+  }
 }
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
index 10024c8..25330e2 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
@@ -29,7 +29,6 @@ import com.datatorrent.api.annotation.ApplicationAnnotation;
 import com.datatorrent.contrib.enrich.JsonFSLoader;
 import com.datatorrent.contrib.enrich.POJOEnricher;
 import com.datatorrent.contrib.parser.JsonParser;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
 
 @ApplicationAnnotation(name = "EnricherAppWithJSONFile")
 /**
@@ -37,10 +36,15 @@ import com.datatorrent.lib.io.ConsoleOutputOperator;
  */
 public class EnricherAppWithJSONFile implements StreamingApplication
 {
+
+
+
+  DataGenerator dataGenerator;
+
   @Override
   public void populateDAG(DAG dag, Configuration conf)
   {
-    DataGenerator dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class);
+    dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class);
     JsonParser parser = dag.addOperator("Parser", JsonParser.class);
 
     /**
@@ -60,10 +64,21 @@ public class EnricherAppWithJSONFile implements StreamingApplication
     enrich.setIncludeFields(includeFields);
     enrich.setLookupFields(lookupFields);
 
-    ConsoleOutputOperator console = dag.addOperator("Console", ConsoleOutputOperator.class);
+    LineOutputOperator fileOut = dag.addOperator("fileOut", LineOutputOperator.class);
+
 
     dag.addStream("Parse", dataGenerator.output, parser.in);
     dag.addStream("Enrich", parser.out, enrich.input);
-    dag.addStream("Console", enrich.output, console.input);
+    dag.addStream("Console", enrich.output, fileOut.input);
+  }
+
+  public DataGenerator getDataGenerator()
+  {
+    return dataGenerator;
+  }
+
+  public void setDataGenerator(DataGenerator dataGenerator)
+  {
+    this.dataGenerator = dataGenerator;
   }
 }
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
index d89986a..4b0ad45 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
@@ -83,12 +83,12 @@ public class POJOEnriched
   @Override
   public String toString()
   {
-    return "POJOEnriched{" +
-        "phone='" + phone + '\'' +
-        ", imei='" + imei + '\'' +
-        ", imsi='" + imsi + '\'' +
-        ", circleId=" + circleId +
-        ", circleName='" + circleName + '\'' +
-        '}';
+    return "{" +
+        "\"phone\":\"" + phone + "\"," +
+        "\"imei\":\"" + imei + "\"," +
+        "\"imsi\":\"" + imsi + "\"," +
+        "\"circleId\":" + circleId + "," +
+        "\"circleName\":\"" + circleName + "\"" +
+        "}";
   }
 }
diff --git a/examples/enricher/src/main/resources/META-INF/properties.xml b/examples/enricher/src/main/resources/META-INF/properties.xml
index c37be45..db581da 100644
--- a/examples/enricher/src/main/resources/META-INF/properties.xml
+++ b/examples/enricher/src/main/resources/META-INF/properties.xml
@@ -37,13 +37,13 @@
   </property>
   <property>
     <name>dt.application.EnricherAppWithJSONFile.operator.Enrich.prop.store.fileName</name>
-    <value>/tmp/circleMapping.txt</value>
+    <value>./src/main/resources/circleMapping.txt</value>
   </property>
 
   <!-- file output operator -->
   <property>
     <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.filePath</name>
-    <value>/tmp/enrichedData</value>
+    <value>./target/enrichedData</value>
   </property>
   <property>
     <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.baseName</name>
@@ -51,11 +51,11 @@
   </property>
   <property>
     <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.maxLength</name>
-    <value>1024</value>
+    <value>102400</value>
   </property>
   <property>
     <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.rotationWindows</name>
-    <value>4</value>
+    <value>0</value>
   </property>
 </configuration>
 
diff --git a/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
index 6cbf846..8f1aed3 100644
--- a/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
+++ b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
@@ -18,14 +18,23 @@
  */
 package org.apache.apex.examples.enricher;
 
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.Test;
 
+import org.apache.apex.api.EmbeddedAppLauncher;
+import org.apache.apex.api.Launcher;
 import org.apache.hadoop.conf.Configuration;
 
-import com.datatorrent.api.LocalMode;
+import com.datatorrent.api.Attribute;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
 
 /**
  * Test the DAG declaration in local mode.
@@ -36,16 +45,77 @@ public class ApplicationTest
   @Test
   public void testApplication() throws Exception
   {
-    try {
-      LocalMode lma = LocalMode.newInstance();
-      Configuration conf = new Configuration(false);
-      conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
-      lma.prepareDAG(new EnricherAppWithJSONFile(), conf);
-      LocalMode.Controller lc = lma.getController();
-      lc.run(10000); // runs for 10 seconds and quits
-    } catch (ConstraintViolationException e) {
-      Assert.fail("constraint violations: " + e.getConstraintViolations());
+    Configuration conf = new Configuration(false);
+    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
+    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
+    launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
+    EnricherAppWithJSONFile enricherAppWithJSONFile = new EnricherAppWithJSONFile();
+    Launcher.AppHandle appHandle = launcher.launchApp(enricherAppWithJSONFile, conf, launchAttributes);
+    File folder = new File("target/enrichedData");
+    File[] listOfFiles = null;
+    int sleepTimeCounterForLoopExit = 0;
+    int sleepTimePerIteration = 5000;
+    // First ensure that the output file is generated and the expected lines match
+    // The test main thread sleeps until the expected lines are generated in the output file
+    while (true) {
+      sleepTimeCounterForLoopExit += sleepTimePerIteration;
+      // break the loop of the test output is not generated by 40 seconds
+      if (sleepTimeCounterForLoopExit > 40000) {
+        break;
+      }
+      Thread.sleep(sleepTimePerIteration);
+      listOfFiles = folder.listFiles();
+      if ( listOfFiles.length == 0) {
+        continue;
+      }
+      String lineRead = null;
+      int counter = 0;
+      try (BufferedReader reader = new BufferedReader(new FileReader(listOfFiles[0].getAbsolutePath()))) {
+        while ((lineRead = reader.readLine()) != null) {
+          counter += 1;
+        }
+      }
+      if (counter < enricherAppWithJSONFile.getDataGenerator().getLimit()) {
+        continue;
+      } else {
+        break;
+      }
     }
+    listOfFiles = folder.listFiles();
+    assertEnrichedOutput(listOfFiles);
   }
 
+  private void assertEnrichedOutput(File[] listOfFiles) throws IOException
+  {
+    // Ensure that only one file is generated as output per properties configuration
+    assertEquals(1, listOfFiles.length);
+    String aJSONEntry = null;
+    ObjectMapper mapper = new ObjectMapper();
+    Map<String,Integer> countsOfEnrichedValues = new HashMap<>();
+    String enrichedValue = null;
+    try (BufferedReader reader = new BufferedReader(new FileReader(listOfFiles[0].getAbsolutePath()))) {
+      while ((aJSONEntry = reader.readLine()) != null) {
+        POJOEnriched anEnrichedEntry = mapper.readValue(aJSONEntry, POJOEnriched.class);
+        enrichedValue = anEnrichedEntry.getCircleName();
+        assertNotNull(enrichedValue); // all values should have an entiched value
+        if (!countsOfEnrichedValues.containsKey(enrichedValue)) {
+          countsOfEnrichedValues.put(enrichedValue,1);
+        } else {
+          Integer currentCountForEnrichedEntry = countsOfEnrichedValues.get(enrichedValue);
+          countsOfEnrichedValues.put(enrichedValue, currentCountForEnrichedEntry + 1);
+        }
+      }
+    }
+    for (String circleName : countsOfEnrichedValues.keySet() ) {
+      int currentCircleNameAsciiCode = ((int)circleName.charAt(0));
+      Integer circleNameCount = countsOfEnrichedValues.get(circleName);
+      // All circle names  "A" to "E" should have a count of 2 and all others should have 1
+      if ( ( currentCircleNameAsciiCode > 64 ) && (currentCircleNameAsciiCode < 70) ) {
+        assertEquals(2,circleNameCount.intValue());
+      } else {
+        assertEquals(1, circleNameCount.intValue());
+      }
+    }
+  }
 }

-- 
To stop receiving notification emails like this one, please contact
"commits@apex.apache.org" <co...@apex.apache.org>.