You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2017/03/27 15:19:05 UTC
[04/19] apex-malhar git commit: SPOI-8728 Json Parser
SPOI-8728 Json Parser
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/8e200974
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/8e200974
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/8e200974
Branch: refs/heads/master
Commit: 8e200974435c30c498760ab669f39a1dbca8b270
Parents: 11f3767
Author: shubham <sh...@github.com>
Authored: Tue Jul 19 16:27:31 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/parser/README.md | 37 +++
examples/parser/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/parser/pom.xml | 309 +++++++++++++++++++
examples/parser/src/assemble/appPackage.xml | 43 +++
.../tutorial/jsonparser/Application.java | 35 +++
.../tutorial/jsonparser/Campaign.java | 74 +++++
.../tutorial/jsonparser/JsonGenerator.java | 83 +++++
.../src/main/resources/META-INF/properties.xml | 40 +++
.../tutorial/jsonparser/ApplicationTest.java | 36 +++
.../parser/src/test/resources/log4j.properties | 22 ++
10 files changed, 723 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/README.md
----------------------------------------------------------------------
diff --git a/examples/parser/README.md b/examples/parser/README.md
new file mode 100644
index 0000000..da230e3
--- /dev/null
+++ b/examples/parser/README.md
@@ -0,0 +1,37 @@
+This project contains applications showcasing different **Parsers** and **Formatters** present in the Malhar Library. For all the apps, parameters are configurable in META_INF/properties.xml.
+
+* **Json Parser App**
+This app showcases **Json Parser**. Data generator sends Json data to the Json Parser which emits each record as POJO on the *output* port. The parser also has *parsedOutput* port that outputs each record as JSONObject and *error* port that emits a error records as key value pair.
+
+* **Csv Parser App**
+
+This application showcases how to use [CsvParser](https://datatorrent.com/docs/apidocs/com/datatorrent/contrib/parser/CsvParser.html) from [Apex Malhar](https://github.com/apache/apex-malhar) library. The CsvParser converts your delimited data to a key-value pair map or concrete java class also know as [POJO](https://en.wikipedia.org/wiki/Plain_Old_Java_Object). The parser emits key-value pair map on *parsedOutput* port. It emits POJO on *out* and error records on *err* port.
+
+Follow these steps to run this application:
+
+**Step 1**: Build the code:
+
+ shell> mvn clean install
+
+**Step 2**: Upload the `target/parser-1.0-SNAPSHOT.apa` to the UI console if available or launch it from
+the commandline using `apex` cli script.
+
+**Step 3**: During launch use `src/main/resources/META-INF/properties-csvParseApplication.xml` as a custom configuration file; then verify
+that the output by checking hdfs file path configured in properties-csvParseApplication.xml
+
+* **Xml Parser App**
+
+This application showcases how to use [XmlParser](https://datatorrent.com/docs/apidocs/com/datatorrent/lib/parser/XmlParser.html)
+from [Apex Malhar](https://github.com/apache/apex-malhar) library. The XmlParser Operator converts XML string to POJO.
+The parser emits dom based Document on *parsedOutput* port. It emits POJO on *out* and error records on *err* port.
+Follow these steps to run this application:
+
+**Step 1**: Build the code:
+
+ shell> mvn clean install
+
+**Step 2**: Upload the `target/parser-1.0-SNAPSHOT.apa` to the UI console if available or launch it from
+the commandline using `apex` cli script.
+
+**Step 3**: During launch use `src/main/resources/META-INF/properties-xmlParseApplication.xml` as a custom configuration file; then verify
+that the output by checking hdfs file path configured in properties-xmlParseApplication.xml
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/parser/XmlJavadocCommentsExtractor.xsl b/examples/parser/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/parser/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/pom.xml
----------------------------------------------------------------------
diff --git a/examples/parser/pom.xml b/examples/parser/pom.xml
new file mode 100644
index 0000000..4eefe75
--- /dev/null
+++ b/examples/parser/pom.xml
@@ -0,0 +1,309 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>parser</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Parser Apps</name>
+ <description>Applications to showcase different parsers</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ <malhar.version>3.7.0-SNAPSHOT</malhar.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${pmalhar.versioroject.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <!--
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${malhar.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.5.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.fge</groupId>
+ <artifactId>json-schema-validator</artifactId>
+ <version>2.0.1</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.supercsv</groupId>
+ <artifactId>super-csv</artifactId>
+ <version>2.4.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.univocity</groupId>
+ <artifactId>univocity-parsers</artifactId>
+ <version>2.0.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/parser/src/assemble/appPackage.xml b/examples/parser/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/parser/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
new file mode 100644
index 0000000..e75541d
--- /dev/null
+++ b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
@@ -0,0 +1,35 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.parser.JsonParser;
+import com.datatorrent.lib.formatter.JsonFormatter;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name = "JsonProcessor")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
+ JsonParser parser = dag.addOperator("JsonParser", JsonParser.class);
+ JsonFormatter formatter = dag.addOperator("JsonFormatter", JsonFormatter.class);
+
+ ConsoleOutputOperator jsonString = dag.addOperator("JsonString", ConsoleOutputOperator.class);
+ ConsoleOutputOperator jsonObject = dag.addOperator("JsonObject", ConsoleOutputOperator.class);
+ ConsoleOutputOperator error = dag.addOperator("Error", ConsoleOutputOperator.class);
+
+ dag.addStream("json", generator.out, parser.in);
+ dag.addStream("pojo", parser.out, formatter.in);
+ dag.addStream("jsonString", formatter.out, jsonString.input);
+ dag.addStream("jsonObject", parser.parsedOutput, jsonObject.input);
+ dag.addStream("error", parser.err, error.input);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
new file mode 100644
index 0000000..f0245eb
--- /dev/null
+++ b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
@@ -0,0 +1,74 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import java.util.Date;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Campaign
+{
+ private int adId;
+ private String campaignName;
+ @JsonProperty("budget")
+ private double campaignBudget;
+ private boolean weatherTargeting;
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy/MM/dd")
+ private Date startDate;
+
+ public int getAdId()
+ {
+ return adId;
+ }
+
+ public void setAdId(int adId)
+ {
+ this.adId = adId;
+ }
+
+ public String getCampaignName()
+ {
+ return campaignName;
+ }
+
+ public void setCampaignName(String campaignName)
+ {
+ this.campaignName = campaignName;
+ }
+
+ public double getCampaignBudget()
+ {
+ return campaignBudget;
+ }
+
+ public void setCampaignBudget(double campaignBudget)
+ {
+ this.campaignBudget = campaignBudget;
+ }
+
+ public boolean isWeatherTargeting()
+ {
+ return weatherTargeting;
+ }
+
+ public void setWeatherTargeting(boolean weatherTargeting)
+ {
+ this.weatherTargeting = weatherTargeting;
+ }
+
+ public Date getStartDate()
+ {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate)
+ {
+ this.startDate = startDate;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Campaign [adId=" + adId + ", campaignName=" + campaignName + ", campaignBudget=" + campaignBudget
+ + ", weatherTargeting=" + weatherTargeting + ", startDate=" + startDate + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
new file mode 100644
index 0000000..90beb13
--- /dev/null
+++ b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
@@ -0,0 +1,83 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+public class JsonGenerator extends BaseOperator implements InputOperator
+{
+
+ private static final Logger LOG = LoggerFactory.getLogger(JsonGenerator.class);
+
+ @Min(1)
+ private int numTuples = 20;
+ private transient int count = 0;
+
+ public static Random rand = new Random();
+ public static SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd");
+ public static int[] adId = { 1, 2, 3, 4, 5 };
+ public static String[] campaignName = { "cmp1", "cmp2", "cmp3", "cmp4" };
+ public static double[] campaignBudget = { 10000.0, 20000.0, 300000.0 };
+ public static boolean[] weatherTargeting = { true, false };
+ private int sleepTime;
+
+ public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
+
+ private static String getNext(int num)
+ {
+
+ JSONObject obj = new JSONObject();
+ try {
+ obj.put("adId", adId[num % adId.length]);
+ obj.put("campaignName", campaignName[num % campaignName.length]);
+ obj.put("campaignBudget", campaignBudget[num % campaignBudget.length]);
+ obj.put("weatherTargeting", weatherTargeting[num % weatherTargeting.length]);
+ obj.put("startDate", sdf.format(new Date()));
+ } catch (JSONException e) {
+ return null;
+ }
+ return obj.toString();
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(getNext(rand.nextInt(numTuples) + 1).getBytes());
+ } else {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("Sleep interrupted");
+ }
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/resources/META-INF/properties.xml b/examples/parser/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..d8ac178
--- /dev/null
+++ b/examples/parser/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<configuration>
+ <!-- <property> <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from
+ the user or custom config when launching)</value> </property> -->
+ <!-- memory assigned to app master <property> <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value> </property> -->
+
+ <!-- these values override those set in populateDAG -->
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonParser.prop.sleepTime
+ </name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonParser.port.out.attr.TUPLE_CLASS
+ </name>
+ <value>com.datatorrent.tutorial.jsonparser.Campaign</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonFormatter.port.in.attr.TUPLE_CLASS
+ </name>
+ <value>com.datatorrent.tutorial.jsonparser.Campaign</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonString.prop.stringFormat
+ </name>
+ <value>JsonString:%s</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonObject.prop.stringFormat
+ </name>
+ <value>JsonObject:%s</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.Error.prop.stringFormat
+ </name>
+ <value>Error:%s</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java b/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
new file mode 100644
index 0000000..390d4cb
--- /dev/null
+++ b/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
@@ -0,0 +1,36 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.tutorial.jsonparser.Application;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10 * 1000); // runs for 30 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/resources/log4j.properties b/examples/parser/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/parser/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info